diff --git a/.gitignore b/.gitignore index b01c05eb..cce85605 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,8 @@ __debug* cover.out covcounters.** covmeta.** +.claude/ +.claude-flow/ +.swarm +.mcp.json +CLAUDE.md \ No newline at end of file diff --git a/Makefile b/Makefile index 64a24330..0b6dd6f9 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ package: package-k3k package-k3k-kubelet ## Package the k3k and k3k-kubelet Dock .PHONY: package-% package-%: - docker build -f package/Dockerfile.$* \ + docker buildx build --platform linux/amd64,linux/arm64 -f package/Dockerfile.$* \ -t $(REPO)/$*:$(VERSION) \ -t $(REPO)/$*:latest \ -t $(REPO)/$*:dev . diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index 35de1c3e..9eee4c0e 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -4,7 +4,6 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 - helm.sh/resource-policy: keep name: clusters.k3k.io spec: group: k3k.io @@ -55,9 +54,11 @@ spec: description: Spec defines the desired state of the Cluster. properties: addons: - description: Addons specifies secrets containing raw YAML to deploy on cluster startup. + description: Addons specifies secrets containing raw YAML to deploy + on cluster startup. items: - description: Addon specifies a Secret containing YAML to be deployed on cluster startup. + description: Addon specifies a Secret containing YAML to be deployed + on cluster startup. properties: secretNamespace: description: SecretNamespace is the namespace of the Secret. @@ -73,7 +74,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -92,17 +94,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -127,14 +132,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -161,7 +168,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -179,7 +187,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -187,14 +196,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -219,14 +230,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -260,7 +273,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -274,10 +288,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -285,14 +301,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -362,14 +381,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -459,14 +481,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -536,14 +560,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -601,7 +627,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -615,10 +643,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -626,14 +656,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -703,14 +736,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -800,14 +836,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -877,14 +915,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -950,9 +990,11 @@ spec: type: string type: array agentEnvs: - description: AgentEnvs specifies list of environment variables to set in the agent pod. + description: AgentEnvs specifies list of environment variables to + set in the agent pod. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment variable present in + a Container. properties: name: description: |- @@ -972,7 +1014,8 @@ spec: Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment variable's value. Cannot + be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -990,7 +1033,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the ConfigMap or its key + must be defined type: boolean required: - key @@ -1002,10 +1046,12 @@ spec: spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to select in the specified + API version. type: string required: - fieldPath @@ -1039,7 +1085,8 @@ spec: Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. + description: The name of the volume mount containing + the env file. type: string required: - key @@ -1053,13 +1100,15 @@ spec: (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required for volumes, + optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output format of the exposed + resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -1073,7 +1122,8 @@ spec: description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret to select from. Must + be a valid secret key. type: string name: default: "" @@ -1085,7 +1135,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the Secret or its key must + be defined type: boolean required: - key @@ -1126,14 +1177,16 @@ spec: - message: clusterDNS is immutable rule: self == oldSelf customCAs: - description: CustomCAs specifies the cert/key pairs for custom CA certificates. + description: CustomCAs specifies the cert/key pairs for custom CA + certificates. properties: enabled: default: true description: Enabled toggles this feature on or off. type: boolean sources: - description: Sources defines the sources for all required custom CA certificates. + description: Sources defines the sources for all required custom + CA certificates. properties: clientCA: description: ClientCA specifies the client-ca cert/key pair. @@ -1148,7 +1201,8 @@ spec: - secretName type: object etcdPeerCA: - description: ETCDPeerCA specifies the etcd-peer-ca cert/key pair. + description: ETCDPeerCA specifies the etcd-peer-ca cert/key + pair. properties: secretName: description: |- @@ -1160,7 +1214,8 @@ spec: - secretName type: object etcdServerCA: - description: ETCDServerCA specifies the etcd-server-ca cert/key pair. + description: ETCDServerCA specifies the etcd-server-ca cert/key + pair. properties: secretName: description: |- @@ -1172,7 +1227,8 @@ spec: - secretName type: object requestHeaderCA: - description: RequestHeaderCA specifies the request-header-ca cert/key pair. + description: RequestHeaderCA specifies the request-header-ca + cert/key pair. properties: secretName: description: |- @@ -1196,7 +1252,8 @@ spec: - secretName type: object serviceAccountToken: - description: ServiceAccountToken specifies the service-account-token key. + description: ServiceAccountToken specifies the service-account-token + key. properties: secretName: description: |- @@ -1225,19 +1282,23 @@ spec: By default, it's only exposed as a ClusterIP. properties: ingress: - description: Ingress specifies options for exposing the API server through an Ingress. + description: Ingress specifies options for exposing the API server + through an Ingress. properties: annotations: additionalProperties: type: string - description: Annotations specifies annotations to add to the Ingress. + description: Annotations specifies annotations to add to the + Ingress. type: object ingressClassName: - description: IngressClassName specifies the IngressClass to use for the Ingress. + description: IngressClassName specifies the IngressClass to + use for the Ingress. type: string type: object loadBalancer: - description: LoadBalancer specifies options for exposing the API server through a LoadBalancer service. + description: LoadBalancer specifies options for exposing the API + server through a LoadBalancer service. properties: etcdPort: description: |- @@ -1255,7 +1316,8 @@ spec: type: integer type: object nodePort: - description: NodePort specifies options for exposing the API server through NodePort. + description: NodePort specifies options for exposing the API server + through NodePort. properties: etcdPort: description: |- @@ -1274,8 +1336,10 @@ spec: type: object type: object x-kubernetes-validations: - - message: ingress, loadbalancer and nodePort are mutually exclusive; only one can be set - rule: '[has(self.ingress), has(self.loadBalancer), has(self.nodePort)].filter(x, x).size() <= 1' + - message: ingress, loadbalancer and nodePort are mutually exclusive; + only one can be set + rule: '[has(self.ingress), has(self.loadBalancer), has(self.nodePort)].filter(x, + x).size() <= 1' hostUsers: description: |- HostUsers sets the user namespace for server and agent pods. @@ -1293,12 +1357,14 @@ spec: - enum: - shared - virtual + - hcp - enum: - shared - virtual + - hcp default: shared description: |- - Mode specifies the cluster provisioning mode: "shared" or "virtual". + Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp". Defaults to "shared". This field is immutable. type: string x-kubernetes-validations: @@ -1414,7 +1480,8 @@ spec: secret contents will be mounted. type: string optional: - description: optional field specify whether the Secret or its keys must be defined + description: optional field specify whether the Secret or its + keys must be defined type: boolean role: description: |- @@ -1556,16 +1623,20 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that applies to + the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that applies to + the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that applies to + the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that applies to + the container. type: string type: object seccompProfile: @@ -1608,7 +1679,8 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name of the GMSA + credential spec to use. type: string hostProcess: description: |- @@ -1632,7 +1704,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1651,17 +1724,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1686,14 +1762,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1720,7 +1798,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -1738,7 +1817,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -1746,14 +1826,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1778,14 +1860,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1819,7 +1903,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1833,10 +1918,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -1844,14 +1931,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1921,14 +2011,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -2018,14 +2111,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2095,14 +2190,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2160,7 +2257,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -2174,10 +2273,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -2185,14 +2286,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -2262,14 +2366,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -2359,14 +2466,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2436,14 +2545,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2509,9 +2620,11 @@ spec: type: string type: array serverEnvs: - description: ServerEnvs specifies list of environment variables to set in the server pod. + description: ServerEnvs specifies list of environment variables to + set in the server pod. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment variable present in + a Container. properties: name: description: |- @@ -2531,7 +2644,8 @@ spec: Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment variable's value. Cannot + be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -2549,7 +2663,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the ConfigMap or its key + must be defined type: boolean required: - key @@ -2561,10 +2676,12 @@ spec: spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to select in the specified + API version. type: string required: - fieldPath @@ -2598,7 +2715,8 @@ spec: Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. + description: The name of the volume mount containing + the env file. type: string required: - key @@ -2612,13 +2730,15 @@ spec: (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required for volumes, + optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output format of the exposed + resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -2632,7 +2752,8 @@ spec: description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret to select from. Must + be a valid secret key. type: string name: default: "" @@ -2644,7 +2765,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the Secret or its key must + be defined type: boolean required: - key @@ -2665,7 +2787,8 @@ spec: description: ServerLimit specifies resource limits for server nodes. type: object serverResources: - description: ServerResources specifies resources limits and requests for server nodes. + description: ServerResources specifies resources limits and requests + for server nodes. properties: claims: description: |- @@ -2744,7 +2867,8 @@ spec: rule: self == oldSelf sync: default: {} - description: Sync specifies the resources types that will be synced from virtual cluster to host cluster. + description: Sync specifies the resources types that will be synced + from virtual cluster to host cluster. properties: configMaps: default: @@ -2885,7 +3009,8 @@ spec: type: object type: object tlsSANs: - description: TLSSANs specifies subject alternative names for the K3s server certificate. + description: TLSSANs specifies subject alternative names for the K3s + server certificate. items: type: string type: array @@ -2895,10 +3020,12 @@ spec: The Secret must have a "token" field in its data. properties: name: - description: name is unique within a namespace to reference a secret resource. + description: name is unique within a namespace to reference a + secret resource. type: string namespace: - description: namespace defines the space within which the secret name must be unique. + description: namespace defines the space within which the secret + name must be unique. type: string type: object x-kubernetes-map-type: atomic @@ -2918,7 +3045,8 @@ spec: description: WorkerLimit specifies resource limits for agent nodes. type: object workerResources: - description: WorkerResources specifies resources limits and requests for worker nodes. + description: WorkerResources specifies resources limits and requests + for worker nodes. properties: claims: description: |- @@ -2988,9 +3116,11 @@ spec: description: ClusterDNS is the IP address for the CoreDNS service. type: string conditions: - description: Conditions are the individual conditions for the cluster set. + description: Conditions are the individual conditions for the cluster + set. items: - description: Condition contains details for one aspect of the current state of this API Resource. + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: description: |- @@ -3043,15 +3173,24 @@ spec: - type type: object type: array + hcpRegistration: + description: |- + HCPRegistration is a copy-pasteable K3s installer command that external + (BYO) nodes can run to register against an HCP-mode cluster. + Only populated when Mode is "hcp" and an externally-routable endpoint + (NodePort, LoadBalancer or Ingress) is configured. + type: string hostVersion: description: HostVersion is the Kubernetes version of the host node. type: string kubeletPort: - description: KubeletPort specefies the port used by k3k-kubelet in shared mode. + description: KubeletPort specefies the port used by k3k-kubelet in + shared mode. type: integer phase: default: Unknown - description: Phase is a high-level summary of the cluster's current lifecycle state. + description: Phase is a high-level summary of the cluster's current + lifecycle state. enum: - Pending - Provisioning @@ -3071,7 +3210,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for + the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3090,17 +3230,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with + the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3125,14 +3268,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3159,7 +3304,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -3177,7 +3323,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -3185,14 +3332,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3217,14 +3366,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3258,7 +3409,9 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3272,10 +3425,13 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -3283,14 +3439,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3360,14 +3519,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3457,14 +3619,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3534,14 +3699,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3599,7 +3767,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3613,10 +3783,13 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -3624,14 +3797,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3701,14 +3877,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3798,14 +3977,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3875,14 +4057,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3948,16 +4133,19 @@ spec: This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. type: boolean name: - description: name is the name of the VirtualClusterPolicy currently applied to this cluster. + description: name is the name of the VirtualClusterPolicy currently + applied to this cluster. minLength: 1 type: string nodeSelector: additionalProperties: type: string - description: nodeSelector is a node selector enforced by the active VirtualClusterPolicy. + description: nodeSelector is a node selector enforced by the active + VirtualClusterPolicy. type: object priorityClass: - description: priorityClass is the priority class enforced by the active VirtualClusterPolicy. + description: priorityClass is the priority class enforced by the + active VirtualClusterPolicy. type: string runtimeClassName: description: |- @@ -4012,14 +4200,16 @@ spec: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX capabilities + type type: string type: array x-kubernetes-list-type: atomic drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX capabilities + type type: string type: array x-kubernetes-list-type: atomic @@ -4081,16 +4271,20 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that applies + to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that applies + to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that applies + to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that applies + to the container. type: string type: object seccompProfile: @@ -4133,7 +4327,8 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. type: string hostProcess: description: |- @@ -4157,7 +4352,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for + the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4176,17 +4372,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with + the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4211,14 +4410,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4245,7 +4446,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -4263,7 +4465,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -4271,14 +4474,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4303,14 +4508,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4344,7 +4551,9 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4358,10 +4567,13 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -4369,14 +4581,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4446,14 +4661,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4543,14 +4761,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -4620,14 +4841,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -4685,7 +4909,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4699,10 +4925,13 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -4710,14 +4939,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4787,14 +5019,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4884,14 +5119,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -4961,14 +5199,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -5171,13 +5412,15 @@ spec: - name type: object policyName: - description: PolicyName specifies the virtual cluster policy name bound to the virtual cluster. + description: PolicyName specifies the virtual cluster policy name + bound to the virtual cluster. type: string serviceCIDR: description: ServiceCIDR is the CIDR range for service IPs. type: string tlsSANs: - description: TLSSANs specifies subject alternative names for the K3s server certificate. + description: TLSSANs specifies subject alternative names for the K3s + server certificate. items: type: string type: array diff --git a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml index a056411b..184c5f98 100644 --- a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml +++ b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml @@ -4,7 +4,6 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 - helm.sh/resource-policy: keep name: virtualclusterpolicies.k3k.io spec: group: k3k.io @@ -50,11 +49,18 @@ spec: description: Spec defines the desired state of the VirtualClusterPolicy. properties: allowedMode: + allOf: + - enum: + - shared + - virtual + - hcp + - enum: + - shared + - virtual + - hcp default: shared - description: AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". - enum: - - shared - - virtual + description: AllowedMode specifies the allowed cluster provisioning + mode. Defaults to "shared". type: string x-kubernetes-validations: - message: mode is immutable @@ -65,7 +71,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -84,17 +91,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -119,14 +129,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -153,7 +165,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -171,7 +184,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -179,14 +193,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -211,14 +227,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -252,7 +270,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -266,10 +285,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -277,14 +298,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -354,14 +378,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -451,14 +478,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -528,14 +557,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -593,7 +624,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -607,10 +640,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -618,14 +653,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -695,14 +733,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -792,14 +833,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -869,14 +912,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -937,10 +982,12 @@ spec: defaultNodeSelector: additionalProperties: type: string - description: DefaultNodeSelector specifies the node selector that applies to all clusters (server + agent) in the target Namespace. + description: DefaultNodeSelector specifies the node selector that + applies to all clusters (server + agent) in the target Namespace. type: object defaultPriorityClass: - description: DefaultPriorityClass specifies the priorityClassName applied to all pods of all clusters in the target Namespace. + description: DefaultPriorityClass specifies the priorityClassName + applied to all pods of all clusters in the target Namespace. type: string defaultServerAffinity: description: |- @@ -948,7 +995,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -967,17 +1015,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1002,14 +1053,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1036,7 +1089,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -1054,7 +1108,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -1062,14 +1117,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1094,14 +1151,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1135,7 +1194,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1149,10 +1209,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -1160,14 +1222,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1237,14 +1302,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1334,14 +1402,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1411,14 +1481,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1476,7 +1548,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1490,10 +1564,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -1501,14 +1577,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1578,14 +1657,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1675,14 +1757,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1752,14 +1836,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1818,7 +1904,8 @@ spec: type: object type: object disableNetworkPolicy: - description: DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. + description: DisableNetworkPolicy indicates whether to disable the + creation of a default network policy for cluster isolation. type: boolean hostUsers: description: |- @@ -1833,9 +1920,11 @@ spec: to set defaults and constraints (min/max) properties: limits: - description: Limits is the list of LimitRangeItem objects that are enforced. + description: Limits is the list of LimitRangeItem objects that + are enforced. items: - description: LimitRangeItem defines a min/max usage limit for any resource that matches on kind. + description: LimitRangeItem defines a min/max usage limit for + any resource that matches on kind. properties: default: additionalProperties: @@ -1844,7 +1933,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Default resource requirement limit value by resource name if resource limit is omitted. + description: Default resource requirement limit value by + resource name if resource limit is omitted. type: object defaultRequest: additionalProperties: @@ -1853,7 +1943,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: DefaultRequest is the default resource requirement request value by resource name if resource request is omitted. + description: DefaultRequest is the default resource requirement + request value by resource name if resource request is + omitted. type: object max: additionalProperties: @@ -1862,7 +1954,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Max usage constraints on this kind by resource name. + description: Max usage constraints on this kind by resource + name. type: object maxLimitRequestRatio: additionalProperties: @@ -1871,7 +1964,11 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: MaxLimitRequestRatio if specified, the named resource must have a request and limit that are both non-zero where limit divided by request is less than or equal to the enumerated value; this represents the max burst for the named resource. + description: MaxLimitRequestRatio if specified, the named + resource must have a request and limit that are both non-zero + where limit divided by request is less than or equal to + the enumerated value; this represents the max burst for + the named resource. type: object min: additionalProperties: @@ -1880,7 +1977,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Min usage constraints on this kind by resource name. + description: Min usage constraints on this kind by resource + name. type: object type: description: Type of resource that this limit applies to. @@ -1894,14 +1992,16 @@ spec: - limits type: object podSecurityAdmissionLevel: - description: PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. + description: PodSecurityAdmissionLevel specifies the pod security + admission level applied to the pods in the namespace. enum: - privileged - baseline - restricted type: string quota: - description: Quota specifies the resource limits for clusters within a clusterpolicy. + description: Quota specifies the resource limits for clusters within + a clusterpolicy. properties: hard: additionalProperties: @@ -1921,7 +2021,8 @@ spec: For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. properties: matchExpressions: - description: A list of scope selector requirements by scope of the resources. + description: A list of scope selector requirements by scope + of the resources. items: description: |- A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator @@ -1933,7 +2034,8 @@ spec: Valid operators are In, NotIn, Exists, DoesNotExist. type: string scopeName: - description: The name of the scope that the selector applies to. + description: The name of the scope that the selector + applies to. type: string values: description: |- @@ -1958,7 +2060,8 @@ spec: A collection of filters that must match each object tracked by a quota. If not specified, the quota matches all objects. items: - description: A ResourceQuotaScope defines a filter that must match each object tracked by a quota + description: A ResourceQuotaScope defines a filter that must + match each object tracked by a quota type: string type: array x-kubernetes-list-type: atomic @@ -2085,16 +2188,20 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that applies to + the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that applies to + the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that applies to + the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that applies to + the container. type: string type: object seccompProfile: @@ -2137,7 +2244,8 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name of the GMSA + credential spec to use. type: string hostProcess: description: |- @@ -2157,7 +2265,8 @@ spec: type: object sync: default: {} - description: Sync specifies the resources types that will be synced from virtual cluster to host cluster. + description: Sync specifies the resources types that will be synced + from virtual cluster to host cluster. properties: configMaps: default: @@ -2302,9 +2411,11 @@ spec: description: Status reflects the observed state of the VirtualClusterPolicy. properties: conditions: - description: Conditions are the individual conditions for the cluster set. + description: Conditions are the individual conditions for the cluster + set. items: - description: Condition contains details for one aspect of the current state of this API Resource. + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: description: |- @@ -2358,10 +2469,12 @@ spec: type: object type: array lastUpdateTime: - description: LastUpdate is the timestamp when the status was last updated. + description: LastUpdate is the timestamp when the status was last + updated. type: string observedGeneration: - description: ObservedGeneration was the generation at the time the status was updated. + description: ObservedGeneration was the generation at the time the + status was updated. format: int64 type: integer summary: diff --git a/cli/cmds/cluster_create.go b/cli/cmds/cluster_create.go index cb6af241..90ae6498 100644 --- a/cli/cmds/cluster_create.go +++ b/cli/cmds/cluster_create.go @@ -84,7 +84,7 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm return errors.New("invalid cluster name") } - if config.mode == string(v1beta1.SharedClusterMode) && config.agents != 0 { + if (config.mode == string(v1beta1.SharedClusterMode) || config.mode == string(v1beta1.HCPClusterMode)) && config.agents != 0 { return errors.New("invalid flag, --agents flag is only allowed in virtual mode") } diff --git a/cli/cmds/cluster_create_flags.go b/cli/cmds/cluster_create_flags.go index 47d2585b..ec6adc28 100644 --- a/cli/cmds/cluster_create_flags.go +++ b/cli/cmds/cluster_create_flags.go @@ -27,7 +27,7 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) { cmd.Flags().StringArrayVar(&cfg.labels, "labels", []string{}, "Labels to add to the cluster object (e.g. key=value)") cmd.Flags().StringArrayVar(&cfg.annotations, "annotations", []string{}, "Annotations to add to the cluster object (e.g. key=value)") cmd.Flags().StringVar(&cfg.version, "version", "", "k3s version") - cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual)") + cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual, hcp)") cmd.Flags().StringVar(&cfg.kubeconfigServerHost, "kubeconfig-server", "", "override the kubeconfig server host") cmd.Flags().StringVar(&cfg.policy, "policy", "", "The policy to create the cluster in") cmd.Flags().StringVar(&cfg.customCertsPath, "custom-certs", "", "The path for custom certificate directory") @@ -54,10 +54,10 @@ func validateCreateConfig(cfg *CreateConfig) error { if cfg.mode != "" { switch cfg.mode { - case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode): + case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode): return nil default: - return errors.New(`mode should be one of "shared" or "virtual"`) + return errors.New(`mode should be one of "shared", "virtual" or "hcp"`) } } diff --git a/cli/cmds/policy_create.go b/cli/cmds/policy_create.go index 7d999d01..fd35e5e9 100644 --- a/cli/cmds/policy_create.go +++ b/cli/cmds/policy_create.go @@ -34,10 +34,10 @@ func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command { Example: "k3kcli policy create [command options] NAME", PreRunE: func(cmd *cobra.Command, args []string) error { switch config.mode { - case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode): + case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode): return nil default: - return errors.New(`mode should be one of "shared" or "virtual"`) + return errors.New(`mode should be one of "shared", "virtual" or "hcp"`) } }, RunE: policyCreateAction(appCtx, config), diff --git a/examples/hcp-server.yaml b/examples/hcp-server.yaml new file mode 100644 index 00000000..8b4ed923 --- /dev/null +++ b/examples/hcp-server.yaml @@ -0,0 +1,10 @@ +apiVersion: k3k.io/v1beta1 +kind: Cluster +metadata: + name: hcp-server +spec: + mode: hcp + servers: 1 + version: v1.33.1-k3s1 + expose: + nodePort: {} diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index 17738d42..a3758fb1 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -45,11 +45,11 @@ type ClusterSpec struct { // +optional Version string `json:"version,omitempty"` - // Mode specifies the cluster provisioning mode: "shared" or "virtual". + // Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp". // Defaults to "shared". This field is immutable. // // +kubebuilder:default="shared" - // +kubebuilder:validation:Enum=shared;virtual + // +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf" // +optional Mode ClusterMode `json:"mode,omitempty"` @@ -413,7 +413,7 @@ type StorageClassSyncConfig struct { // ClusterMode is the possible provisioning mode of a Cluster. // -// +kubebuilder:validation:Enum=shared;virtual +// +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:default="shared" type ClusterMode string @@ -423,6 +423,11 @@ const ( // VirtualClusterMode represents a cluster that runs in a virtual environment. VirtualClusterMode = ClusterMode("virtual") + + // HCPClusterMode represents a Hosted Control Plane: an agentless K3s control + // plane managed by k3k inside the host cluster. End users join their own + // external nodes (BYO) using the standard K3s installer command. + HCPClusterMode = ClusterMode("hcp") ) // PersistenceMode is the storage mode of a Cluster. @@ -619,6 +624,14 @@ type ClusterStatus struct { // +optional KubeletPort int `json:"kubeletPort,omitempty"` + // HCPRegistration is a copy-pasteable K3s installer command that external + // (BYO) nodes can run to register against an HCP-mode cluster. + // Only populated when Mode is "hcp" and an externally-routable endpoint + // (NodePort, LoadBalancer or Ingress) is configured. + // + // +optional + HCPRegistration string `json:"hcpRegistration,omitempty"` + // Conditions are the individual conditions for the cluster set. // // +optional @@ -774,6 +787,7 @@ type VirtualClusterPolicySpec struct { // AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". // // +kubebuilder:default=shared + // +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf" // +optional AllowedMode ClusterMode `json:"allowedMode,omitempty"` diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index ba3946dc..dad6447e 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -400,9 +400,10 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus } } - // in virtual mode assign a default serviceCIDR - if cluster.Spec.Mode == v1beta1.VirtualClusterMode { - log.V(1).Info("assign default service CIDR for virtual mode") + // virtual and hcp modes both run a self-contained K3s control plane and + // need their own pod/service CIDR independent of the host cluster. + if cluster.Spec.Mode == v1beta1.VirtualClusterMode || cluster.Spec.Mode == v1beta1.HCPClusterMode { + log.V(1).Info("assign default service CIDR", "mode", cluster.Spec.Mode) cluster.Status.ServiceCIDR = defaultVirtualServiceCIDR } @@ -447,6 +448,21 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus return err } + // In hcp mode, derive the K3s installer command end-users run on their + // external nodes and surface it on the Cluster status. We also own the + // default/kubernetes Endpoints inside the virtual cluster (the apiserver + // reconciler is disabled for HCP) so external-node pods can reach the + // in-cluster apiserver ClusterIP. + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + if err := c.ensureHCPRegistration(ctx, cluster, token); err != nil { + return err + } + + if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil { + return err + } + } + // Important: if you need to call the Server API of the Virtual Cluster // this needs to be done AFTER he kubeconfig has been generated @@ -896,6 +912,13 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet } func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error { + // hcp mode is BYO-node by design: external (out-of-host-cluster) nodes join + // using the standard K3s installer command surfaced via Status.HCPRegistration. + // k3k therefore does not provision any agent pods on the host cluster. + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + return nil + } + config := agent.NewConfig(cluster, c.Client, c.Scheme) var agentEnsurer agent.ResourceEnsurer diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go new file mode 100644 index 00000000..8d82cd8b --- /dev/null +++ b/pkg/controller/cluster/hcp.go @@ -0,0 +1,216 @@ +package cluster + +import ( + "context" + "fmt" + "net" + "net/url" + "strconv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + "github.com/rancher/k3k/pkg/controller/cluster/server" +) + +// endpointSliceSkipMirrorLabel is the upstream label that opts an Endpoints +// object out of the kube-controller-manager EndpointSlice mirroring controller. +// The kube-apiserver normally sets it on default/kubernetes (because it +// manages EndpointSlices itself); in HCP mode we want the mirror controller +// to handle slices, so we strip the label. +const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror" + +// ensureHCPRegistration computes the K3s installer command external nodes can +// run to join an HCP-mode cluster and stores it on cluster.Status.HCPRegistration. +// +// When the cluster's Service is not externally reachable (no NodePort, +// LoadBalancer or Ingress configured) the command cannot be built; in that +// case the Ready condition is set to False with reason HCPNoExternalEndpoint +// so the operator surfaces the problem without failing the reconciliation. +func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error { + log := ctrl.LoggerFrom(ctx) + + url, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0) + if err != nil { + return err + } + + if !external { + log.Info("HCP cluster has no externally-routable endpoint; skipping registration command", + "cluster", cluster.Name, "namespace", cluster.Namespace) + + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: ConditionReady, + Status: metav1.ConditionFalse, + Reason: ReasonHCPNoExternalEndpoint, + Message: "HCP cluster has no external endpoint; set spec.expose.nodePort, spec.expose.loadBalancer or spec.expose.ingress so external nodes can reach the API server", + }) + + cluster.Status.HCPRegistration = "" + + return nil + } + + version := cluster.Spec.Version + if version == "" { + version = cluster.Status.HostVersion + } + + cluster.Status.HCPRegistration = hcpRegistrationCommand(version, url, token) + + return nil +} + +// hcpRegistrationCommand returns the standard K3s installer one-liner an +// end-user can copy onto an external host to join an HCP cluster. +func hcpRegistrationCommand(version, serverURL, token string) string { + if version == "" { + return fmt.Sprintf("curl -sfL https://get.k3s.io | K3S_URL=%s K3S_TOKEN=%s sh -", serverURL, token) + } + + return fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s K3S_URL=%s K3S_TOKEN=%s sh -", + version, serverURL, token) +} + +// ensureHCPKubernetesEndpoints maintains the default/kubernetes Service +// Endpoints inside the virtual cluster, pointing them at the externally +// reachable host:port (NodePort / LoadBalancer / Ingress) so that pods +// scheduled on external worker nodes can reach the in-cluster apiserver +// ClusterIP. +// +// Background: the kube-apiserver normally reconciles default/kubernetes +// Endpoints to its own --advertise-address:--secure-port (the host-cluster +// pod IP and 6443). External worker nodes have no route to the host-cluster +// pod CIDR, so kube-proxy DNAT to that endpoint fails. We disable the +// apiserver reconciler in HCP mode (see serverOptions) and own this +// Endpoints object instead. +func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error { + log := ctrl.LoggerFrom(ctx) + + rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0) + if err != nil { + return err + } + + if !external { + // ensureHCPRegistration already surfaces this via Ready=False; + // nothing for us to do here. + return nil + } + + host, port, err := parseHCPHostPort(rawURL) + if err != nil { + return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err) + } + + addr, err := hcpEndpointAddress(host) + if err != nil { + return err + } + + virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace) + if err != nil { + return fmt.Errorf("creating virtual cluster client: %w", err) + } + + endpoints := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubernetes", + Namespace: metav1.NamespaceDefault, + }, + } + + _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error { + // Allow EndpointSlice mirroring; the apiserver may have set + // skip-mirror=true before we disabled its endpoint reconciler. + if endpoints.Labels != nil { + delete(endpoints.Labels, endpointSliceSkipMirrorLabel) + } + + endpoints.Subsets = []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{addr}, + Ports: []corev1.EndpointPort{ + { + Name: "https", + Port: port, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + + return nil + }) + if err != nil { + return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err) + } + + log.V(1).Info("HCP kubernetes endpoints reconciled", + "address", addr.IP, "hostname", addr.Hostname, "port", port) + + return nil +} + +// parseHCPHostPort extracts the host and port from a server URL produced by +// server.ServerURL. The port defaults to 443 when omitted. +func parseHCPHostPort(rawURL string) (string, int32, error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", 0, err + } + + host := u.Hostname() + if host == "" { + return "", 0, fmt.Errorf("missing host in URL %q", rawURL) + } + + portStr := u.Port() + + var port int32 = 443 + if portStr != "" { + p, err := strconv.Atoi(portStr) + if err != nil { + return "", 0, fmt.Errorf("invalid port in URL %q: %w", rawURL, err) + } + + if p <= 0 || p > 65535 { + return "", 0, fmt.Errorf("port %d out of range in URL %q", p, rawURL) + } + + port = int32(p) + } + + return host, port, nil +} + +// hcpEndpointAddress builds a corev1.EndpointAddress from the externally +// reachable host. Endpoints require an IP; if the host is a DNS name we +// resolve it and keep the original name as Hostname so logs/events remain +// human-readable. +func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { + if ip := net.ParseIP(host); ip != nil { + return corev1.EndpointAddress{IP: host}, nil + } + + ips, err := net.LookupIP(host) + if err != nil { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err) + } + + for _, ip := range ips { + if v4 := ip.To4(); v4 != nil { + return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil + } + } + + if len(ips) == 0 { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no IPs", host) + } + + return corev1.EndpointAddress{IP: ips[0].String(), Hostname: host}, nil +} diff --git a/pkg/controller/cluster/hcp_test.go b/pkg/controller/cluster/hcp_test.go new file mode 100644 index 00000000..a7edb8a9 --- /dev/null +++ b/pkg/controller/cluster/hcp_test.go @@ -0,0 +1,200 @@ +package cluster + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + "github.com/rancher/k3k/pkg/controller" + "github.com/rancher/k3k/pkg/controller/cluster/server" +) + +func Test_hcpRegistrationCommand(t *testing.T) { + tests := []struct { + name string + version string + serverURL string + token string + want string + }{ + { + name: "with version", + version: "v1.33.1-k3s1", + serverURL: "https://1.2.3.4:30443", + token: "abcd1234", + want: "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.33.1-k3s1 K3S_URL=https://1.2.3.4:30443 K3S_TOKEN=abcd1234 sh -", + }, + { + name: "without version", + version: "", + serverURL: "https://hcp.example.com", + token: "tok", + want: "curl -sfL https://get.k3s.io | K3S_URL=https://hcp.example.com K3S_TOKEN=tok sh -", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := hcpRegistrationCommand(tt.version, tt.serverURL, tt.token) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_ensureHCPRegistration(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + + cluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "team-a"}, + Spec: v1beta1.ClusterSpec{ + Mode: v1beta1.HCPClusterMode, + Version: "v1.33.1-k3s1", + TLSSANs: []string{"hcp.example.com"}, + }, + Status: v1beta1.ClusterStatus{ + TLSSANs: []string{"hcp.example.com"}, + }, + } + + t.Run("nodeport service produces ready-to-copy command", func(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: server.ServiceName(cluster.Name), + Namespace: cluster.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + ClusterIP: "10.43.0.50", + Ports: []corev1.ServicePort{ + {Name: "k3s-server-port", Port: 443, NodePort: 31001}, + }, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build() + r := &ClusterReconciler{Client: fakeClient} + + c := cluster.DeepCopy() + require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "join-token-xyz")) + + assert.Contains(t, c.Status.HCPRegistration, "K3S_URL=https://hcp.example.com:31001") + assert.Contains(t, c.Status.HCPRegistration, "K3S_TOKEN=join-token-xyz") + assert.Contains(t, c.Status.HCPRegistration, "INSTALL_K3S_VERSION=v1.33.1-k3s1") + assert.True(t, strings.HasPrefix(c.Status.HCPRegistration, "curl -sfL https://get.k3s.io")) + }) + + t.Run("clusterip-only service sets degraded condition and clears registration", func(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: server.ServiceName(cluster.Name), + Namespace: cluster.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "10.43.0.50", + Ports: []corev1.ServicePort{ + {Name: "k3s-server-port", Port: 443}, + }, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build() + r := &ClusterReconciler{Client: fakeClient} + + c := cluster.DeepCopy() + c.Status.HCPRegistration = "stale-value" + require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "ignored")) + + assert.Empty(t, c.Status.HCPRegistration) + + cond := meta.FindStatusCondition(c.Status.Conditions, ConditionReady) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, ReasonHCPNoExternalEndpoint, cond.Reason) + }) +} + +func Test_parseHCPHostPort(t *testing.T) { + tests := []struct { + name string + url string + wantHost string + wantPort int32 + wantErr bool + }{ + { + name: "ip with explicit port", + url: "https://10.144.101.195:30337", + wantHost: "10.144.101.195", + wantPort: 30337, + }, + { + name: "hostname without port defaults to 443", + url: "https://hcp.example.com", + wantHost: "hcp.example.com", + wantPort: 443, + }, + { + name: "hostname with explicit port", + url: "https://hcp.example.com:6443", + wantHost: "hcp.example.com", + wantPort: 6443, + }, + { + name: "missing host", + url: "https://", + wantErr: true, + }, + { + name: "non-numeric port", + url: "https://host:abc", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + host, port, err := parseHCPHostPort(tt.url) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.wantHost, host) + assert.Equal(t, tt.wantPort, port) + }) + } +} + +func Test_hcpEndpointAddress(t *testing.T) { + t.Run("ipv4 literal is passed through", func(t *testing.T) { + got, err := hcpEndpointAddress("10.144.101.195") + require.NoError(t, err) + assert.Equal(t, "10.144.101.195", got.IP) + assert.Empty(t, got.Hostname) + }) + + t.Run("unresolvable hostname errors", func(t *testing.T) { + _, err := hcpEndpointAddress("definitely-not-a-real-host.invalid") + require.Error(t, err) + }) +} + +// Compile-time assertion: every reused exported name from the controller +// package below this test file must remain stable. If `controller.K3SImage` +// disappears (refactor), this guards the dependency. +var _ = controller.K3SImage diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index 52136afd..25aafd8f 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -11,7 +11,6 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" - "github.com/rancher/k3k/pkg/controller/cluster/agent" ) // serverConfig are few options from k3s server options that will @@ -77,10 +76,51 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok serverConfig.Server = "https://" + serviceIP } - if cluster.Spec.Mode != agent.VirtualNodeMode { - serverConfig.DisableAgent = true - serverConfig.EgressSelectorMode = "disabled" - serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"} + // shared and hcp modes both run K3s with --disable-agent (agentless server). + // hcp additionally relies on this to satisfy the PRD requirement that the + // control plane never runs a kubelet and is not enumerated as a node. + if cluster.Spec.Mode != v1beta1.VirtualClusterMode { + opts = opts + "disable-agent: true\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage\n" + } + + // In shared mode workloads run on the host cluster, so the apiserver pod + // can reach them directly via the host pod network and the egress + // selector is unnecessary. + // + // In hcp mode the apiserver pod has NO route to the virtual cluster's + // pod CIDR (which only exists on joined external worker nodes), and the + // kube-apiserver bypasses kube-proxy when calling webhooks / proxying + // to pods: it resolves Service -> Endpoints itself and dials the Pod IP + // directly. We therefore tunnel apiserver egress through the WebSocket + // each k3s-agent maintains back to the server. + // + // We pick "cluster" rather than "pod" or "agent" because the agent-side + // authorizer differs by mode (k3s pkg/agent/tunnel/tunnel.go): + // - agent: only kubelet calls are tunneled; pod-IP dials go direct + // and fail in HCP (no route to virtual pod CIDR). + // - pod: authorizer only allows pod IPs the agent has *already + // watched*. A newly-created pod's IP is rejected with + // "connect not allowed", which terminates the entire + // remotedialer session and 502s in-flight kubelet streams + // -> kubectl logs / exec / webhooks become flaky. + // - cluster: authorizer pre-populates the cluster CIDR + node IPs as + // non-hostNet entries, so every pod IP and every node port + // is permitted. No race, no per-port allowlist. This is + // what we want for a managed control plane. + switch cluster.Spec.Mode { + case v1beta1.SharedClusterMode: + opts = opts + "egress-selector-mode: disabled\n" + case v1beta1.HCPClusterMode: + opts = opts + "egress-selector-mode: cluster\n" + } + + // In hcp mode the apiserver pod IP is unreachable from external worker + // nodes, so the kube-apiserver's default lease-based endpoint reconciler + // would publish a broken default/kubernetes Endpoints (advertise-address + + // secure-port). Disable it so K3k can own that Endpoints object and point + // it at the externally-reachable host:port (NodePort / LB / Ingress). + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + opts = opts + "kube-apiserver-arg:\n- endpoint-reconciler-type=none\n" } return serverConfig diff --git a/pkg/controller/cluster/server/endpoint.go b/pkg/controller/cluster/server/endpoint.go new file mode 100644 index 00000000..13066304 --- /dev/null +++ b/pkg/controller/cluster/server/endpoint.go @@ -0,0 +1,111 @@ +package server + +import ( + "context" + "fmt" + "slices" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" +) + +// ServerURL returns the URL at which the K3s API server of a virtual cluster +// is reachable. The second return value reports whether that URL is routable +// from outside the host cluster (true for NodePort/LoadBalancer/Ingress, false +// for plain ClusterIP exposition). +// +// hostServerIP is used as the address when the underlying Service is a +// NodePort. serverPort, when non-zero, overrides the port discovered from the +// Service. +func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, bool, error) { + key := types.NamespacedName{ + Name: ServiceName(cluster.Name), + Namespace: cluster.Namespace, + } + + var k3kService corev1.Service + if err := c.Get(ctx, key, &k3kService); err != nil { + return "", false, err + } + + ip := k3kService.Spec.ClusterIP + port := int32(httpsPort) + external := false + + if len(k3kService.Spec.Ports) == 0 { + logrus.Warn("No ports exposed by the cluster service.") + } + + switch k3kService.Spec.Type { + case corev1.ServiceTypeNodePort: + ip = hostServerIP + external = true + + if len(k3kService.Spec.Ports) > 0 { + port = k3kService.Spec.Ports[0].NodePort + } + case corev1.ServiceTypeLoadBalancer: + external = true + + if len(k3kService.Status.LoadBalancer.Ingress) > 0 { + ip = k3kService.Status.LoadBalancer.Ingress[0].IP + } else { + logrus.Warn("No ingress found in LoadBalancer service.") + } + + if len(k3kService.Spec.Ports) > 0 { + port = k3kService.Spec.Ports[0].Port + } + } + + if serverPort != 0 { + port = int32(serverPort) + } + + if !slices.Contains(cluster.Status.TLSSANs, ip) { + logrus.Warnf("IP %s not in tlsSANs.", ip) + + if len(cluster.Spec.TLSSANs) > 0 { + logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0]) + + ip = cluster.Spec.TLSSANs[0] + } else if len(cluster.Status.TLSSANs) > 0 { + logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0]) + + ip = cluster.Status.TLSSANs[0] + } else { + logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.") + } + } + + url := "https://" + ip + if port != httpsPort { + url = fmt.Sprintf("%s:%d", url, port) + } + + // if ingress is specified, use the ingress host + if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { + var k3kIngress networkingv1.Ingress + + ingressKey := types.NamespacedName{ + Name: IngressName(cluster.Name), + Namespace: cluster.Namespace, + } + + if err := c.Get(ctx, ingressKey, &k3kIngress); err != nil { + return "", external, err + } + + if len(k3kIngress.Spec.Rules) > 0 { + url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host) + external = true + } + } + + return url, external, nil +} diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 330ceefa..8ea304cb 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -22,7 +22,6 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" - "github.com/rancher/k3k/pkg/controller/cluster/agent" "github.com/rancher/k3k/pkg/controller/cluster/mounts" ) @@ -250,8 +249,10 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo }, }, } - // start the pod unprivileged in shared mode - if s.mode == agent.VirtualNodeMode { + // virtual mode runs an embedded kubelet inside the server pod and therefore + // requires Privileged. shared and hcp modes are agentless (no kubelet) and + // run unprivileged. + if s.mode == string(v1beta1.VirtualClusterMode) { podSpec.Containers[0].SecurityContext = &corev1.SecurityContext{ Privileged: ptr.To(true), } diff --git a/pkg/controller/cluster/server/template.go b/pkg/controller/cluster/server/template.go index daf83699..774fb985 100644 --- a/pkg/controller/cluster/server/template.go +++ b/pkg/controller/cluster/server/template.go @@ -19,7 +19,7 @@ safe_mode() { CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip) fi - if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ {{.K3K_MODE}} != "virtual" ]; then + if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" = "shared" ] || [ "{{.K3K_MODE}}" = "hcp" ]; then return fi @@ -116,7 +116,8 @@ configure_cgroups() { fi # only configure the cgroups if the runtime used is the default and the mode is virtual - if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then + # shared and hcp run agentless (no kubelet) and don't need cgroup overrides. + if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then return fi diff --git a/pkg/controller/cluster/status.go b/pkg/controller/cluster/status.go index 0a229f77..cff1024d 100644 --- a/pkg/controller/cluster/status.go +++ b/pkg/controller/cluster/status.go @@ -19,11 +19,12 @@ const ( ConditionReady = "Ready" // Condition Reasons - ReasonValidationFailed = "ValidationFailed" - ReasonProvisioning = "Provisioning" - ReasonProvisioned = "Provisioned" - ReasonProvisioningFailed = "ProvisioningFailed" - ReasonTerminating = "Terminating" + ReasonValidationFailed = "ValidationFailed" + ReasonProvisioning = "Provisioning" + ReasonProvisioned = "Provisioned" + ReasonProvisioningFailed = "ProvisioningFailed" + ReasonTerminating = "Terminating" + ReasonHCPNoExternalEndpoint = "HCPNoExternalEndpoint" ) func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.Cluster, reconcileErr error) { diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go index 44860314..86ecf9e3 100644 --- a/pkg/controller/kubeconfig/kubeconfig.go +++ b/pkg/controller/kubeconfig/kubeconfig.go @@ -3,18 +3,12 @@ package kubeconfig import ( "context" "crypto/x509" - "fmt" - "slices" "time" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/user" "sigs.k8s.io/controller-runtime/pkg/client" certutil "github.com/rancher/dynamiclistener/cert" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" @@ -60,7 +54,7 @@ func (k *KubeConfig) Generate(ctx context.Context, client client.Client, cluster return nil, err } - url, err := getURLFromService(ctx, client, cluster, hostServerIP, port) + url, _, err := server.ServerURL(ctx, client, cluster, hostServerIP, port) if err != nil { return nil, err } @@ -93,84 +87,3 @@ func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi return config } -func getURLFromService(ctx context.Context, client client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, error) { - // get the server service to extract the right IP - key := types.NamespacedName{ - Name: server.ServiceName(cluster.Name), - Namespace: cluster.Namespace, - } - - var k3kService corev1.Service - if err := client.Get(ctx, key, &k3kService); err != nil { - return "", err - } - - ip := k3kService.Spec.ClusterIP - port := int32(443) - - if len(k3kService.Spec.Ports) == 0 { - logrus.Warn("No ports exposed by the cluster service.") - } - - switch k3kService.Spec.Type { - case corev1.ServiceTypeNodePort: - ip = hostServerIP - - if len(k3kService.Spec.Ports) > 0 { - port = k3kService.Spec.Ports[0].NodePort - } - case corev1.ServiceTypeLoadBalancer: - if len(k3kService.Status.LoadBalancer.Ingress) > 0 { - ip = k3kService.Status.LoadBalancer.Ingress[0].IP - } else { - logrus.Warn("No ingress found in LoadBalancer service.") - } - - if len(k3kService.Spec.Ports) > 0 { - port = k3kService.Spec.Ports[0].Port - } - } - - if serverPort != 0 { - port = int32(serverPort) - } - - if !slices.Contains(cluster.Status.TLSSANs, ip) { - logrus.Warnf("IP %s not in tlsSANs.", ip) - - if len(cluster.Spec.TLSSANs) > 0 { - logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0]) - - ip = cluster.Spec.TLSSANs[0] - } else if len(cluster.Status.TLSSANs) > 0 { - logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0]) - - ip = cluster.Status.TLSSANs[0] - } else { - logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.") - } - } - - url := "https://" + ip - if port != 443 { - url = fmt.Sprintf("%s:%d", url, port) - } - - // if ingress is specified, use the ingress host - if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { - var k3kIngress networkingv1.Ingress - - ingressKey := types.NamespacedName{ - Name: server.IngressName(cluster.Name), - Namespace: cluster.Namespace, - } - - if err := client.Get(ctx, ingressKey, &k3kIngress); err != nil { - return "", err - } - - url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host) - } - - return url, nil -} diff --git a/scripts/generate b/scripts/generate index 820d958f..5b618ca4 100755 --- a/scripts/generate +++ b/scripts/generate @@ -15,5 +15,5 @@ go run sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSIO # add the 'helm.sh/resource-policy: keep' annotation to the CRDs for f in ./charts/k3k/templates/crds/*.yaml; do echo "Annotating $f" - yq -c -i '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$f" + #yq -c -i '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$f" done diff --git a/tests/e2e/common_test.go b/tests/e2e/common_test.go index 67668724..ed44c396 100644 --- a/tests/e2e/common_test.go +++ b/tests/e2e/common_test.go @@ -288,7 +288,8 @@ func (c *VirtualCluster) NewNginxPod(namespace string) (*corev1.Pod, string) { By(fmt.Sprintf("Nginx Pod is running (%s/%s)", nginxPod.Namespace, nginxPod.Name)) - // only check the pod on the host cluster if the mode is shared mode + // only check the pod on the host cluster if the mode is shared mode. + // hcp is agentless and BYO-node, so no host-side pod mirror exists. if c.Cluster.Spec.Mode != v1beta1.SharedClusterMode { return nginxPod, "" }