From 67d5f4dbfc4f5dddb538493b4c1b00617b47dae2 Mon Sep 17 00:00:00 2001 From: jpgouin Date: Mon, 18 May 2026 17:08:05 +0200 Subject: [PATCH 01/36] Add HCP (Hosted Control Plane) support Introduce hosted control plane mode for k3k virtual clusters, including API types, controller logic, server endpoint handling, CLI flags, CRD updates, kubeconfig generation, and examples. Co-Authored-By: RuFlo --- .gitignore | 5 + Makefile | 2 +- .../k3k/templates/crds/k3k.io_clusters.yaml | 649 ++++++++++++------ .../crds/k3k.io_virtualclusterpolicies.yaml | 309 ++++++--- cli/cmds/cluster_create.go | 2 +- cli/cmds/cluster_create_flags.go | 6 +- cli/cmds/policy_create.go | 4 +- examples/hcp-server.yaml | 10 + pkg/apis/k3k.io/v1beta1/types.go | 20 +- pkg/controller/cluster/cluster.go | 29 +- pkg/controller/cluster/hcp.go | 216 ++++++ pkg/controller/cluster/hcp_test.go | 200 ++++++ pkg/controller/cluster/server/config.go | 50 +- pkg/controller/cluster/server/endpoint.go | 111 +++ pkg/controller/cluster/server/server.go | 7 +- pkg/controller/cluster/server/template.go | 5 +- pkg/controller/cluster/status.go | 11 +- pkg/controller/kubeconfig/kubeconfig.go | 89 +-- scripts/generate | 2 +- tests/e2e/common_test.go | 3 +- 20 files changed, 1311 insertions(+), 419 deletions(-) create mode 100644 examples/hcp-server.yaml create mode 100644 pkg/controller/cluster/hcp.go create mode 100644 pkg/controller/cluster/hcp_test.go create mode 100644 pkg/controller/cluster/server/endpoint.go diff --git a/.gitignore b/.gitignore index b01c05eb..cce85605 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,8 @@ __debug* cover.out covcounters.** covmeta.** +.claude/ +.claude-flow/ +.swarm +.mcp.json +CLAUDE.md \ No newline at end of file diff --git a/Makefile b/Makefile index 64a24330..0b6dd6f9 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ package: package-k3k package-k3k-kubelet ## Package the k3k and k3k-kubelet Dock .PHONY: package-% package-%: - docker build -f package/Dockerfile.$* \ + docker buildx build --platform linux/amd64,linux/arm64 -f package/Dockerfile.$* \ -t $(REPO)/$*:$(VERSION) \ -t $(REPO)/$*:latest \ -t $(REPO)/$*:dev . diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index 35de1c3e..9eee4c0e 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -4,7 +4,6 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 - helm.sh/resource-policy: keep name: clusters.k3k.io spec: group: k3k.io @@ -55,9 +54,11 @@ spec: description: Spec defines the desired state of the Cluster. properties: addons: - description: Addons specifies secrets containing raw YAML to deploy on cluster startup. + description: Addons specifies secrets containing raw YAML to deploy + on cluster startup. items: - description: Addon specifies a Secret containing YAML to be deployed on cluster startup. + description: Addon specifies a Secret containing YAML to be deployed + on cluster startup. properties: secretNamespace: description: SecretNamespace is the namespace of the Secret. @@ -73,7 +74,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -92,17 +94,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -127,14 +132,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -161,7 +168,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -179,7 +187,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -187,14 +196,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -219,14 +230,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -260,7 +273,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -274,10 +288,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -285,14 +301,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -362,14 +381,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -459,14 +481,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -536,14 +560,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -601,7 +627,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -615,10 +643,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -626,14 +656,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -703,14 +736,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -800,14 +836,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -877,14 +915,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -950,9 +990,11 @@ spec: type: string type: array agentEnvs: - description: AgentEnvs specifies list of environment variables to set in the agent pod. + description: AgentEnvs specifies list of environment variables to + set in the agent pod. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment variable present in + a Container. properties: name: description: |- @@ -972,7 +1014,8 @@ spec: Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment variable's value. Cannot + be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -990,7 +1033,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the ConfigMap or its key + must be defined type: boolean required: - key @@ -1002,10 +1046,12 @@ spec: spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to select in the specified + API version. type: string required: - fieldPath @@ -1039,7 +1085,8 @@ spec: Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. + description: The name of the volume mount containing + the env file. type: string required: - key @@ -1053,13 +1100,15 @@ spec: (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required for volumes, + optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output format of the exposed + resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -1073,7 +1122,8 @@ spec: description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret to select from. Must + be a valid secret key. type: string name: default: "" @@ -1085,7 +1135,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the Secret or its key must + be defined type: boolean required: - key @@ -1126,14 +1177,16 @@ spec: - message: clusterDNS is immutable rule: self == oldSelf customCAs: - description: CustomCAs specifies the cert/key pairs for custom CA certificates. + description: CustomCAs specifies the cert/key pairs for custom CA + certificates. properties: enabled: default: true description: Enabled toggles this feature on or off. type: boolean sources: - description: Sources defines the sources for all required custom CA certificates. + description: Sources defines the sources for all required custom + CA certificates. properties: clientCA: description: ClientCA specifies the client-ca cert/key pair. @@ -1148,7 +1201,8 @@ spec: - secretName type: object etcdPeerCA: - description: ETCDPeerCA specifies the etcd-peer-ca cert/key pair. + description: ETCDPeerCA specifies the etcd-peer-ca cert/key + pair. properties: secretName: description: |- @@ -1160,7 +1214,8 @@ spec: - secretName type: object etcdServerCA: - description: ETCDServerCA specifies the etcd-server-ca cert/key pair. + description: ETCDServerCA specifies the etcd-server-ca cert/key + pair. properties: secretName: description: |- @@ -1172,7 +1227,8 @@ spec: - secretName type: object requestHeaderCA: - description: RequestHeaderCA specifies the request-header-ca cert/key pair. + description: RequestHeaderCA specifies the request-header-ca + cert/key pair. properties: secretName: description: |- @@ -1196,7 +1252,8 @@ spec: - secretName type: object serviceAccountToken: - description: ServiceAccountToken specifies the service-account-token key. + description: ServiceAccountToken specifies the service-account-token + key. properties: secretName: description: |- @@ -1225,19 +1282,23 @@ spec: By default, it's only exposed as a ClusterIP. properties: ingress: - description: Ingress specifies options for exposing the API server through an Ingress. + description: Ingress specifies options for exposing the API server + through an Ingress. properties: annotations: additionalProperties: type: string - description: Annotations specifies annotations to add to the Ingress. + description: Annotations specifies annotations to add to the + Ingress. type: object ingressClassName: - description: IngressClassName specifies the IngressClass to use for the Ingress. + description: IngressClassName specifies the IngressClass to + use for the Ingress. type: string type: object loadBalancer: - description: LoadBalancer specifies options for exposing the API server through a LoadBalancer service. + description: LoadBalancer specifies options for exposing the API + server through a LoadBalancer service. properties: etcdPort: description: |- @@ -1255,7 +1316,8 @@ spec: type: integer type: object nodePort: - description: NodePort specifies options for exposing the API server through NodePort. + description: NodePort specifies options for exposing the API server + through NodePort. properties: etcdPort: description: |- @@ -1274,8 +1336,10 @@ spec: type: object type: object x-kubernetes-validations: - - message: ingress, loadbalancer and nodePort are mutually exclusive; only one can be set - rule: '[has(self.ingress), has(self.loadBalancer), has(self.nodePort)].filter(x, x).size() <= 1' + - message: ingress, loadbalancer and nodePort are mutually exclusive; + only one can be set + rule: '[has(self.ingress), has(self.loadBalancer), has(self.nodePort)].filter(x, + x).size() <= 1' hostUsers: description: |- HostUsers sets the user namespace for server and agent pods. @@ -1293,12 +1357,14 @@ spec: - enum: - shared - virtual + - hcp - enum: - shared - virtual + - hcp default: shared description: |- - Mode specifies the cluster provisioning mode: "shared" or "virtual". + Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp". Defaults to "shared". This field is immutable. type: string x-kubernetes-validations: @@ -1414,7 +1480,8 @@ spec: secret contents will be mounted. type: string optional: - description: optional field specify whether the Secret or its keys must be defined + description: optional field specify whether the Secret or its + keys must be defined type: boolean role: description: |- @@ -1556,16 +1623,20 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that applies to + the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that applies to + the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that applies to + the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that applies to + the container. type: string type: object seccompProfile: @@ -1608,7 +1679,8 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name of the GMSA + credential spec to use. type: string hostProcess: description: |- @@ -1632,7 +1704,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1651,17 +1724,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1686,14 +1762,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1720,7 +1798,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -1738,7 +1817,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -1746,14 +1826,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1778,14 +1860,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1819,7 +1903,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1833,10 +1918,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -1844,14 +1931,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1921,14 +2011,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -2018,14 +2111,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2095,14 +2190,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2160,7 +2257,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -2174,10 +2273,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -2185,14 +2286,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -2262,14 +2366,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -2359,14 +2466,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2436,14 +2545,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2509,9 +2620,11 @@ spec: type: string type: array serverEnvs: - description: ServerEnvs specifies list of environment variables to set in the server pod. + description: ServerEnvs specifies list of environment variables to + set in the server pod. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment variable present in + a Container. properties: name: description: |- @@ -2531,7 +2644,8 @@ spec: Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment variable's value. Cannot + be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -2549,7 +2663,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the ConfigMap or its key + must be defined type: boolean required: - key @@ -2561,10 +2676,12 @@ spec: spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to select in the specified + API version. type: string required: - fieldPath @@ -2598,7 +2715,8 @@ spec: Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. + description: The name of the volume mount containing + the env file. type: string required: - key @@ -2612,13 +2730,15 @@ spec: (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required for volumes, + optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output format of the exposed + resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -2632,7 +2752,8 @@ spec: description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret to select from. Must + be a valid secret key. type: string name: default: "" @@ -2644,7 +2765,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the Secret or its key must + be defined type: boolean required: - key @@ -2665,7 +2787,8 @@ spec: description: ServerLimit specifies resource limits for server nodes. type: object serverResources: - description: ServerResources specifies resources limits and requests for server nodes. + description: ServerResources specifies resources limits and requests + for server nodes. properties: claims: description: |- @@ -2744,7 +2867,8 @@ spec: rule: self == oldSelf sync: default: {} - description: Sync specifies the resources types that will be synced from virtual cluster to host cluster. + description: Sync specifies the resources types that will be synced + from virtual cluster to host cluster. properties: configMaps: default: @@ -2885,7 +3009,8 @@ spec: type: object type: object tlsSANs: - description: TLSSANs specifies subject alternative names for the K3s server certificate. + description: TLSSANs specifies subject alternative names for the K3s + server certificate. items: type: string type: array @@ -2895,10 +3020,12 @@ spec: The Secret must have a "token" field in its data. properties: name: - description: name is unique within a namespace to reference a secret resource. + description: name is unique within a namespace to reference a + secret resource. type: string namespace: - description: namespace defines the space within which the secret name must be unique. + description: namespace defines the space within which the secret + name must be unique. type: string type: object x-kubernetes-map-type: atomic @@ -2918,7 +3045,8 @@ spec: description: WorkerLimit specifies resource limits for agent nodes. type: object workerResources: - description: WorkerResources specifies resources limits and requests for worker nodes. + description: WorkerResources specifies resources limits and requests + for worker nodes. properties: claims: description: |- @@ -2988,9 +3116,11 @@ spec: description: ClusterDNS is the IP address for the CoreDNS service. type: string conditions: - description: Conditions are the individual conditions for the cluster set. + description: Conditions are the individual conditions for the cluster + set. items: - description: Condition contains details for one aspect of the current state of this API Resource. + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: description: |- @@ -3043,15 +3173,24 @@ spec: - type type: object type: array + hcpRegistration: + description: |- + HCPRegistration is a copy-pasteable K3s installer command that external + (BYO) nodes can run to register against an HCP-mode cluster. + Only populated when Mode is "hcp" and an externally-routable endpoint + (NodePort, LoadBalancer or Ingress) is configured. + type: string hostVersion: description: HostVersion is the Kubernetes version of the host node. type: string kubeletPort: - description: KubeletPort specefies the port used by k3k-kubelet in shared mode. + description: KubeletPort specefies the port used by k3k-kubelet in + shared mode. type: integer phase: default: Unknown - description: Phase is a high-level summary of the cluster's current lifecycle state. + description: Phase is a high-level summary of the cluster's current + lifecycle state. enum: - Pending - Provisioning @@ -3071,7 +3210,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for + the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3090,17 +3230,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with + the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3125,14 +3268,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3159,7 +3304,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -3177,7 +3323,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -3185,14 +3332,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3217,14 +3366,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3258,7 +3409,9 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3272,10 +3425,13 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -3283,14 +3439,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3360,14 +3519,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3457,14 +3619,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3534,14 +3699,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3599,7 +3767,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3613,10 +3783,13 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -3624,14 +3797,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3701,14 +3877,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3798,14 +3977,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3875,14 +4057,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3948,16 +4133,19 @@ spec: This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. type: boolean name: - description: name is the name of the VirtualClusterPolicy currently applied to this cluster. + description: name is the name of the VirtualClusterPolicy currently + applied to this cluster. minLength: 1 type: string nodeSelector: additionalProperties: type: string - description: nodeSelector is a node selector enforced by the active VirtualClusterPolicy. + description: nodeSelector is a node selector enforced by the active + VirtualClusterPolicy. type: object priorityClass: - description: priorityClass is the priority class enforced by the active VirtualClusterPolicy. + description: priorityClass is the priority class enforced by the + active VirtualClusterPolicy. type: string runtimeClassName: description: |- @@ -4012,14 +4200,16 @@ spec: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX capabilities + type type: string type: array x-kubernetes-list-type: atomic drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX capabilities + type type: string type: array x-kubernetes-list-type: atomic @@ -4081,16 +4271,20 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that applies + to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that applies + to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that applies + to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that applies + to the container. type: string type: object seccompProfile: @@ -4133,7 +4327,8 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. type: string hostProcess: description: |- @@ -4157,7 +4352,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for + the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4176,17 +4372,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with + the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4211,14 +4410,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4245,7 +4446,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -4263,7 +4465,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -4271,14 +4474,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4303,14 +4508,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4344,7 +4551,9 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4358,10 +4567,13 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -4369,14 +4581,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4446,14 +4661,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4543,14 +4761,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -4620,14 +4841,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -4685,7 +4909,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4699,10 +4925,13 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -4710,14 +4939,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4787,14 +5019,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4884,14 +5119,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -4961,14 +5199,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -5171,13 +5412,15 @@ spec: - name type: object policyName: - description: PolicyName specifies the virtual cluster policy name bound to the virtual cluster. + description: PolicyName specifies the virtual cluster policy name + bound to the virtual cluster. type: string serviceCIDR: description: ServiceCIDR is the CIDR range for service IPs. type: string tlsSANs: - description: TLSSANs specifies subject alternative names for the K3s server certificate. + description: TLSSANs specifies subject alternative names for the K3s + server certificate. items: type: string type: array diff --git a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml index a056411b..184c5f98 100644 --- a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml +++ b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml @@ -4,7 +4,6 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 - helm.sh/resource-policy: keep name: virtualclusterpolicies.k3k.io spec: group: k3k.io @@ -50,11 +49,18 @@ spec: description: Spec defines the desired state of the VirtualClusterPolicy. properties: allowedMode: + allOf: + - enum: + - shared + - virtual + - hcp + - enum: + - shared + - virtual + - hcp default: shared - description: AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". - enum: - - shared - - virtual + description: AllowedMode specifies the allowed cluster provisioning + mode. Defaults to "shared". type: string x-kubernetes-validations: - message: mode is immutable @@ -65,7 +71,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -84,17 +91,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -119,14 +129,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -153,7 +165,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -171,7 +184,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -179,14 +193,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -211,14 +227,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -252,7 +270,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -266,10 +285,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -277,14 +298,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -354,14 +378,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -451,14 +478,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -528,14 +557,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -593,7 +624,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -607,10 +640,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -618,14 +653,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -695,14 +733,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -792,14 +833,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -869,14 +912,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -937,10 +982,12 @@ spec: defaultNodeSelector: additionalProperties: type: string - description: DefaultNodeSelector specifies the node selector that applies to all clusters (server + agent) in the target Namespace. + description: DefaultNodeSelector specifies the node selector that + applies to all clusters (server + agent) in the target Namespace. type: object defaultPriorityClass: - description: DefaultPriorityClass specifies the priorityClassName applied to all pods of all clusters in the target Namespace. + description: DefaultPriorityClass specifies the priorityClassName + applied to all pods of all clusters in the target Namespace. type: string defaultServerAffinity: description: |- @@ -948,7 +995,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -967,17 +1015,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1002,14 +1053,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1036,7 +1089,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -1054,7 +1108,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -1062,14 +1117,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1094,14 +1151,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1135,7 +1194,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1149,10 +1209,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -1160,14 +1222,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1237,14 +1302,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1334,14 +1402,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1411,14 +1481,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1476,7 +1548,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1490,10 +1564,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -1501,14 +1577,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1578,14 +1657,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1675,14 +1757,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1752,14 +1836,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1818,7 +1904,8 @@ spec: type: object type: object disableNetworkPolicy: - description: DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. + description: DisableNetworkPolicy indicates whether to disable the + creation of a default network policy for cluster isolation. type: boolean hostUsers: description: |- @@ -1833,9 +1920,11 @@ spec: to set defaults and constraints (min/max) properties: limits: - description: Limits is the list of LimitRangeItem objects that are enforced. + description: Limits is the list of LimitRangeItem objects that + are enforced. items: - description: LimitRangeItem defines a min/max usage limit for any resource that matches on kind. + description: LimitRangeItem defines a min/max usage limit for + any resource that matches on kind. properties: default: additionalProperties: @@ -1844,7 +1933,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Default resource requirement limit value by resource name if resource limit is omitted. + description: Default resource requirement limit value by + resource name if resource limit is omitted. type: object defaultRequest: additionalProperties: @@ -1853,7 +1943,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: DefaultRequest is the default resource requirement request value by resource name if resource request is omitted. + description: DefaultRequest is the default resource requirement + request value by resource name if resource request is + omitted. type: object max: additionalProperties: @@ -1862,7 +1954,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Max usage constraints on this kind by resource name. + description: Max usage constraints on this kind by resource + name. type: object maxLimitRequestRatio: additionalProperties: @@ -1871,7 +1964,11 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: MaxLimitRequestRatio if specified, the named resource must have a request and limit that are both non-zero where limit divided by request is less than or equal to the enumerated value; this represents the max burst for the named resource. + description: MaxLimitRequestRatio if specified, the named + resource must have a request and limit that are both non-zero + where limit divided by request is less than or equal to + the enumerated value; this represents the max burst for + the named resource. type: object min: additionalProperties: @@ -1880,7 +1977,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Min usage constraints on this kind by resource name. + description: Min usage constraints on this kind by resource + name. type: object type: description: Type of resource that this limit applies to. @@ -1894,14 +1992,16 @@ spec: - limits type: object podSecurityAdmissionLevel: - description: PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. + description: PodSecurityAdmissionLevel specifies the pod security + admission level applied to the pods in the namespace. enum: - privileged - baseline - restricted type: string quota: - description: Quota specifies the resource limits for clusters within a clusterpolicy. + description: Quota specifies the resource limits for clusters within + a clusterpolicy. properties: hard: additionalProperties: @@ -1921,7 +2021,8 @@ spec: For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. properties: matchExpressions: - description: A list of scope selector requirements by scope of the resources. + description: A list of scope selector requirements by scope + of the resources. items: description: |- A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator @@ -1933,7 +2034,8 @@ spec: Valid operators are In, NotIn, Exists, DoesNotExist. type: string scopeName: - description: The name of the scope that the selector applies to. + description: The name of the scope that the selector + applies to. type: string values: description: |- @@ -1958,7 +2060,8 @@ spec: A collection of filters that must match each object tracked by a quota. If not specified, the quota matches all objects. items: - description: A ResourceQuotaScope defines a filter that must match each object tracked by a quota + description: A ResourceQuotaScope defines a filter that must + match each object tracked by a quota type: string type: array x-kubernetes-list-type: atomic @@ -2085,16 +2188,20 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that applies to + the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that applies to + the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that applies to + the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that applies to + the container. type: string type: object seccompProfile: @@ -2137,7 +2244,8 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name of the GMSA + credential spec to use. type: string hostProcess: description: |- @@ -2157,7 +2265,8 @@ spec: type: object sync: default: {} - description: Sync specifies the resources types that will be synced from virtual cluster to host cluster. + description: Sync specifies the resources types that will be synced + from virtual cluster to host cluster. properties: configMaps: default: @@ -2302,9 +2411,11 @@ spec: description: Status reflects the observed state of the VirtualClusterPolicy. properties: conditions: - description: Conditions are the individual conditions for the cluster set. + description: Conditions are the individual conditions for the cluster + set. items: - description: Condition contains details for one aspect of the current state of this API Resource. + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: description: |- @@ -2358,10 +2469,12 @@ spec: type: object type: array lastUpdateTime: - description: LastUpdate is the timestamp when the status was last updated. + description: LastUpdate is the timestamp when the status was last + updated. type: string observedGeneration: - description: ObservedGeneration was the generation at the time the status was updated. + description: ObservedGeneration was the generation at the time the + status was updated. format: int64 type: integer summary: diff --git a/cli/cmds/cluster_create.go b/cli/cmds/cluster_create.go index cb6af241..90ae6498 100644 --- a/cli/cmds/cluster_create.go +++ b/cli/cmds/cluster_create.go @@ -84,7 +84,7 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm return errors.New("invalid cluster name") } - if config.mode == string(v1beta1.SharedClusterMode) && config.agents != 0 { + if (config.mode == string(v1beta1.SharedClusterMode) || config.mode == string(v1beta1.HCPClusterMode)) && config.agents != 0 { return errors.New("invalid flag, --agents flag is only allowed in virtual mode") } diff --git a/cli/cmds/cluster_create_flags.go b/cli/cmds/cluster_create_flags.go index 47d2585b..ec6adc28 100644 --- a/cli/cmds/cluster_create_flags.go +++ b/cli/cmds/cluster_create_flags.go @@ -27,7 +27,7 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) { cmd.Flags().StringArrayVar(&cfg.labels, "labels", []string{}, "Labels to add to the cluster object (e.g. key=value)") cmd.Flags().StringArrayVar(&cfg.annotations, "annotations", []string{}, "Annotations to add to the cluster object (e.g. key=value)") cmd.Flags().StringVar(&cfg.version, "version", "", "k3s version") - cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual)") + cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual, hcp)") cmd.Flags().StringVar(&cfg.kubeconfigServerHost, "kubeconfig-server", "", "override the kubeconfig server host") cmd.Flags().StringVar(&cfg.policy, "policy", "", "The policy to create the cluster in") cmd.Flags().StringVar(&cfg.customCertsPath, "custom-certs", "", "The path for custom certificate directory") @@ -54,10 +54,10 @@ func validateCreateConfig(cfg *CreateConfig) error { if cfg.mode != "" { switch cfg.mode { - case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode): + case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode): return nil default: - return errors.New(`mode should be one of "shared" or "virtual"`) + return errors.New(`mode should be one of "shared", "virtual" or "hcp"`) } } diff --git a/cli/cmds/policy_create.go b/cli/cmds/policy_create.go index 7d999d01..fd35e5e9 100644 --- a/cli/cmds/policy_create.go +++ b/cli/cmds/policy_create.go @@ -34,10 +34,10 @@ func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command { Example: "k3kcli policy create [command options] NAME", PreRunE: func(cmd *cobra.Command, args []string) error { switch config.mode { - case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode): + case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode): return nil default: - return errors.New(`mode should be one of "shared" or "virtual"`) + return errors.New(`mode should be one of "shared", "virtual" or "hcp"`) } }, RunE: policyCreateAction(appCtx, config), diff --git a/examples/hcp-server.yaml b/examples/hcp-server.yaml new file mode 100644 index 00000000..8b4ed923 --- /dev/null +++ b/examples/hcp-server.yaml @@ -0,0 +1,10 @@ +apiVersion: k3k.io/v1beta1 +kind: Cluster +metadata: + name: hcp-server +spec: + mode: hcp + servers: 1 + version: v1.33.1-k3s1 + expose: + nodePort: {} diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index 17738d42..a3758fb1 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -45,11 +45,11 @@ type ClusterSpec struct { // +optional Version string `json:"version,omitempty"` - // Mode specifies the cluster provisioning mode: "shared" or "virtual". + // Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp". // Defaults to "shared". This field is immutable. // // +kubebuilder:default="shared" - // +kubebuilder:validation:Enum=shared;virtual + // +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf" // +optional Mode ClusterMode `json:"mode,omitempty"` @@ -413,7 +413,7 @@ type StorageClassSyncConfig struct { // ClusterMode is the possible provisioning mode of a Cluster. // -// +kubebuilder:validation:Enum=shared;virtual +// +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:default="shared" type ClusterMode string @@ -423,6 +423,11 @@ const ( // VirtualClusterMode represents a cluster that runs in a virtual environment. VirtualClusterMode = ClusterMode("virtual") + + // HCPClusterMode represents a Hosted Control Plane: an agentless K3s control + // plane managed by k3k inside the host cluster. End users join their own + // external nodes (BYO) using the standard K3s installer command. + HCPClusterMode = ClusterMode("hcp") ) // PersistenceMode is the storage mode of a Cluster. @@ -619,6 +624,14 @@ type ClusterStatus struct { // +optional KubeletPort int `json:"kubeletPort,omitempty"` + // HCPRegistration is a copy-pasteable K3s installer command that external + // (BYO) nodes can run to register against an HCP-mode cluster. + // Only populated when Mode is "hcp" and an externally-routable endpoint + // (NodePort, LoadBalancer or Ingress) is configured. + // + // +optional + HCPRegistration string `json:"hcpRegistration,omitempty"` + // Conditions are the individual conditions for the cluster set. // // +optional @@ -774,6 +787,7 @@ type VirtualClusterPolicySpec struct { // AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". // // +kubebuilder:default=shared + // +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf" // +optional AllowedMode ClusterMode `json:"allowedMode,omitempty"` diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index ba3946dc..dad6447e 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -400,9 +400,10 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus } } - // in virtual mode assign a default serviceCIDR - if cluster.Spec.Mode == v1beta1.VirtualClusterMode { - log.V(1).Info("assign default service CIDR for virtual mode") + // virtual and hcp modes both run a self-contained K3s control plane and + // need their own pod/service CIDR independent of the host cluster. + if cluster.Spec.Mode == v1beta1.VirtualClusterMode || cluster.Spec.Mode == v1beta1.HCPClusterMode { + log.V(1).Info("assign default service CIDR", "mode", cluster.Spec.Mode) cluster.Status.ServiceCIDR = defaultVirtualServiceCIDR } @@ -447,6 +448,21 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus return err } + // In hcp mode, derive the K3s installer command end-users run on their + // external nodes and surface it on the Cluster status. We also own the + // default/kubernetes Endpoints inside the virtual cluster (the apiserver + // reconciler is disabled for HCP) so external-node pods can reach the + // in-cluster apiserver ClusterIP. + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + if err := c.ensureHCPRegistration(ctx, cluster, token); err != nil { + return err + } + + if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil { + return err + } + } + // Important: if you need to call the Server API of the Virtual Cluster // this needs to be done AFTER he kubeconfig has been generated @@ -896,6 +912,13 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet } func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error { + // hcp mode is BYO-node by design: external (out-of-host-cluster) nodes join + // using the standard K3s installer command surfaced via Status.HCPRegistration. + // k3k therefore does not provision any agent pods on the host cluster. + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + return nil + } + config := agent.NewConfig(cluster, c.Client, c.Scheme) var agentEnsurer agent.ResourceEnsurer diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go new file mode 100644 index 00000000..8d82cd8b --- /dev/null +++ b/pkg/controller/cluster/hcp.go @@ -0,0 +1,216 @@ +package cluster + +import ( + "context" + "fmt" + "net" + "net/url" + "strconv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + "github.com/rancher/k3k/pkg/controller/cluster/server" +) + +// endpointSliceSkipMirrorLabel is the upstream label that opts an Endpoints +// object out of the kube-controller-manager EndpointSlice mirroring controller. +// The kube-apiserver normally sets it on default/kubernetes (because it +// manages EndpointSlices itself); in HCP mode we want the mirror controller +// to handle slices, so we strip the label. +const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror" + +// ensureHCPRegistration computes the K3s installer command external nodes can +// run to join an HCP-mode cluster and stores it on cluster.Status.HCPRegistration. +// +// When the cluster's Service is not externally reachable (no NodePort, +// LoadBalancer or Ingress configured) the command cannot be built; in that +// case the Ready condition is set to False with reason HCPNoExternalEndpoint +// so the operator surfaces the problem without failing the reconciliation. +func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error { + log := ctrl.LoggerFrom(ctx) + + url, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0) + if err != nil { + return err + } + + if !external { + log.Info("HCP cluster has no externally-routable endpoint; skipping registration command", + "cluster", cluster.Name, "namespace", cluster.Namespace) + + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: ConditionReady, + Status: metav1.ConditionFalse, + Reason: ReasonHCPNoExternalEndpoint, + Message: "HCP cluster has no external endpoint; set spec.expose.nodePort, spec.expose.loadBalancer or spec.expose.ingress so external nodes can reach the API server", + }) + + cluster.Status.HCPRegistration = "" + + return nil + } + + version := cluster.Spec.Version + if version == "" { + version = cluster.Status.HostVersion + } + + cluster.Status.HCPRegistration = hcpRegistrationCommand(version, url, token) + + return nil +} + +// hcpRegistrationCommand returns the standard K3s installer one-liner an +// end-user can copy onto an external host to join an HCP cluster. +func hcpRegistrationCommand(version, serverURL, token string) string { + if version == "" { + return fmt.Sprintf("curl -sfL https://get.k3s.io | K3S_URL=%s K3S_TOKEN=%s sh -", serverURL, token) + } + + return fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s K3S_URL=%s K3S_TOKEN=%s sh -", + version, serverURL, token) +} + +// ensureHCPKubernetesEndpoints maintains the default/kubernetes Service +// Endpoints inside the virtual cluster, pointing them at the externally +// reachable host:port (NodePort / LoadBalancer / Ingress) so that pods +// scheduled on external worker nodes can reach the in-cluster apiserver +// ClusterIP. +// +// Background: the kube-apiserver normally reconciles default/kubernetes +// Endpoints to its own --advertise-address:--secure-port (the host-cluster +// pod IP and 6443). External worker nodes have no route to the host-cluster +// pod CIDR, so kube-proxy DNAT to that endpoint fails. We disable the +// apiserver reconciler in HCP mode (see serverOptions) and own this +// Endpoints object instead. +func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error { + log := ctrl.LoggerFrom(ctx) + + rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0) + if err != nil { + return err + } + + if !external { + // ensureHCPRegistration already surfaces this via Ready=False; + // nothing for us to do here. + return nil + } + + host, port, err := parseHCPHostPort(rawURL) + if err != nil { + return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err) + } + + addr, err := hcpEndpointAddress(host) + if err != nil { + return err + } + + virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace) + if err != nil { + return fmt.Errorf("creating virtual cluster client: %w", err) + } + + endpoints := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubernetes", + Namespace: metav1.NamespaceDefault, + }, + } + + _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error { + // Allow EndpointSlice mirroring; the apiserver may have set + // skip-mirror=true before we disabled its endpoint reconciler. + if endpoints.Labels != nil { + delete(endpoints.Labels, endpointSliceSkipMirrorLabel) + } + + endpoints.Subsets = []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{addr}, + Ports: []corev1.EndpointPort{ + { + Name: "https", + Port: port, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + + return nil + }) + if err != nil { + return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err) + } + + log.V(1).Info("HCP kubernetes endpoints reconciled", + "address", addr.IP, "hostname", addr.Hostname, "port", port) + + return nil +} + +// parseHCPHostPort extracts the host and port from a server URL produced by +// server.ServerURL. The port defaults to 443 when omitted. +func parseHCPHostPort(rawURL string) (string, int32, error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", 0, err + } + + host := u.Hostname() + if host == "" { + return "", 0, fmt.Errorf("missing host in URL %q", rawURL) + } + + portStr := u.Port() + + var port int32 = 443 + if portStr != "" { + p, err := strconv.Atoi(portStr) + if err != nil { + return "", 0, fmt.Errorf("invalid port in URL %q: %w", rawURL, err) + } + + if p <= 0 || p > 65535 { + return "", 0, fmt.Errorf("port %d out of range in URL %q", p, rawURL) + } + + port = int32(p) + } + + return host, port, nil +} + +// hcpEndpointAddress builds a corev1.EndpointAddress from the externally +// reachable host. Endpoints require an IP; if the host is a DNS name we +// resolve it and keep the original name as Hostname so logs/events remain +// human-readable. +func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { + if ip := net.ParseIP(host); ip != nil { + return corev1.EndpointAddress{IP: host}, nil + } + + ips, err := net.LookupIP(host) + if err != nil { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err) + } + + for _, ip := range ips { + if v4 := ip.To4(); v4 != nil { + return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil + } + } + + if len(ips) == 0 { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no IPs", host) + } + + return corev1.EndpointAddress{IP: ips[0].String(), Hostname: host}, nil +} diff --git a/pkg/controller/cluster/hcp_test.go b/pkg/controller/cluster/hcp_test.go new file mode 100644 index 00000000..a7edb8a9 --- /dev/null +++ b/pkg/controller/cluster/hcp_test.go @@ -0,0 +1,200 @@ +package cluster + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + "github.com/rancher/k3k/pkg/controller" + "github.com/rancher/k3k/pkg/controller/cluster/server" +) + +func Test_hcpRegistrationCommand(t *testing.T) { + tests := []struct { + name string + version string + serverURL string + token string + want string + }{ + { + name: "with version", + version: "v1.33.1-k3s1", + serverURL: "https://1.2.3.4:30443", + token: "abcd1234", + want: "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.33.1-k3s1 K3S_URL=https://1.2.3.4:30443 K3S_TOKEN=abcd1234 sh -", + }, + { + name: "without version", + version: "", + serverURL: "https://hcp.example.com", + token: "tok", + want: "curl -sfL https://get.k3s.io | K3S_URL=https://hcp.example.com K3S_TOKEN=tok sh -", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := hcpRegistrationCommand(tt.version, tt.serverURL, tt.token) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_ensureHCPRegistration(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + + cluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "team-a"}, + Spec: v1beta1.ClusterSpec{ + Mode: v1beta1.HCPClusterMode, + Version: "v1.33.1-k3s1", + TLSSANs: []string{"hcp.example.com"}, + }, + Status: v1beta1.ClusterStatus{ + TLSSANs: []string{"hcp.example.com"}, + }, + } + + t.Run("nodeport service produces ready-to-copy command", func(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: server.ServiceName(cluster.Name), + Namespace: cluster.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + ClusterIP: "10.43.0.50", + Ports: []corev1.ServicePort{ + {Name: "k3s-server-port", Port: 443, NodePort: 31001}, + }, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build() + r := &ClusterReconciler{Client: fakeClient} + + c := cluster.DeepCopy() + require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "join-token-xyz")) + + assert.Contains(t, c.Status.HCPRegistration, "K3S_URL=https://hcp.example.com:31001") + assert.Contains(t, c.Status.HCPRegistration, "K3S_TOKEN=join-token-xyz") + assert.Contains(t, c.Status.HCPRegistration, "INSTALL_K3S_VERSION=v1.33.1-k3s1") + assert.True(t, strings.HasPrefix(c.Status.HCPRegistration, "curl -sfL https://get.k3s.io")) + }) + + t.Run("clusterip-only service sets degraded condition and clears registration", func(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: server.ServiceName(cluster.Name), + Namespace: cluster.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "10.43.0.50", + Ports: []corev1.ServicePort{ + {Name: "k3s-server-port", Port: 443}, + }, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build() + r := &ClusterReconciler{Client: fakeClient} + + c := cluster.DeepCopy() + c.Status.HCPRegistration = "stale-value" + require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "ignored")) + + assert.Empty(t, c.Status.HCPRegistration) + + cond := meta.FindStatusCondition(c.Status.Conditions, ConditionReady) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, ReasonHCPNoExternalEndpoint, cond.Reason) + }) +} + +func Test_parseHCPHostPort(t *testing.T) { + tests := []struct { + name string + url string + wantHost string + wantPort int32 + wantErr bool + }{ + { + name: "ip with explicit port", + url: "https://10.144.101.195:30337", + wantHost: "10.144.101.195", + wantPort: 30337, + }, + { + name: "hostname without port defaults to 443", + url: "https://hcp.example.com", + wantHost: "hcp.example.com", + wantPort: 443, + }, + { + name: "hostname with explicit port", + url: "https://hcp.example.com:6443", + wantHost: "hcp.example.com", + wantPort: 6443, + }, + { + name: "missing host", + url: "https://", + wantErr: true, + }, + { + name: "non-numeric port", + url: "https://host:abc", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + host, port, err := parseHCPHostPort(tt.url) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.wantHost, host) + assert.Equal(t, tt.wantPort, port) + }) + } +} + +func Test_hcpEndpointAddress(t *testing.T) { + t.Run("ipv4 literal is passed through", func(t *testing.T) { + got, err := hcpEndpointAddress("10.144.101.195") + require.NoError(t, err) + assert.Equal(t, "10.144.101.195", got.IP) + assert.Empty(t, got.Hostname) + }) + + t.Run("unresolvable hostname errors", func(t *testing.T) { + _, err := hcpEndpointAddress("definitely-not-a-real-host.invalid") + require.Error(t, err) + }) +} + +// Compile-time assertion: every reused exported name from the controller +// package below this test file must remain stable. If `controller.K3SImage` +// disappears (refactor), this guards the dependency. +var _ = controller.K3SImage diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index 52136afd..25aafd8f 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -11,7 +11,6 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" - "github.com/rancher/k3k/pkg/controller/cluster/agent" ) // serverConfig are few options from k3s server options that will @@ -77,10 +76,51 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok serverConfig.Server = "https://" + serviceIP } - if cluster.Spec.Mode != agent.VirtualNodeMode { - serverConfig.DisableAgent = true - serverConfig.EgressSelectorMode = "disabled" - serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"} + // shared and hcp modes both run K3s with --disable-agent (agentless server). + // hcp additionally relies on this to satisfy the PRD requirement that the + // control plane never runs a kubelet and is not enumerated as a node. + if cluster.Spec.Mode != v1beta1.VirtualClusterMode { + opts = opts + "disable-agent: true\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage\n" + } + + // In shared mode workloads run on the host cluster, so the apiserver pod + // can reach them directly via the host pod network and the egress + // selector is unnecessary. + // + // In hcp mode the apiserver pod has NO route to the virtual cluster's + // pod CIDR (which only exists on joined external worker nodes), and the + // kube-apiserver bypasses kube-proxy when calling webhooks / proxying + // to pods: it resolves Service -> Endpoints itself and dials the Pod IP + // directly. We therefore tunnel apiserver egress through the WebSocket + // each k3s-agent maintains back to the server. + // + // We pick "cluster" rather than "pod" or "agent" because the agent-side + // authorizer differs by mode (k3s pkg/agent/tunnel/tunnel.go): + // - agent: only kubelet calls are tunneled; pod-IP dials go direct + // and fail in HCP (no route to virtual pod CIDR). + // - pod: authorizer only allows pod IPs the agent has *already + // watched*. A newly-created pod's IP is rejected with + // "connect not allowed", which terminates the entire + // remotedialer session and 502s in-flight kubelet streams + // -> kubectl logs / exec / webhooks become flaky. + // - cluster: authorizer pre-populates the cluster CIDR + node IPs as + // non-hostNet entries, so every pod IP and every node port + // is permitted. No race, no per-port allowlist. This is + // what we want for a managed control plane. + switch cluster.Spec.Mode { + case v1beta1.SharedClusterMode: + opts = opts + "egress-selector-mode: disabled\n" + case v1beta1.HCPClusterMode: + opts = opts + "egress-selector-mode: cluster\n" + } + + // In hcp mode the apiserver pod IP is unreachable from external worker + // nodes, so the kube-apiserver's default lease-based endpoint reconciler + // would publish a broken default/kubernetes Endpoints (advertise-address + + // secure-port). Disable it so K3k can own that Endpoints object and point + // it at the externally-reachable host:port (NodePort / LB / Ingress). + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + opts = opts + "kube-apiserver-arg:\n- endpoint-reconciler-type=none\n" } return serverConfig diff --git a/pkg/controller/cluster/server/endpoint.go b/pkg/controller/cluster/server/endpoint.go new file mode 100644 index 00000000..13066304 --- /dev/null +++ b/pkg/controller/cluster/server/endpoint.go @@ -0,0 +1,111 @@ +package server + +import ( + "context" + "fmt" + "slices" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" +) + +// ServerURL returns the URL at which the K3s API server of a virtual cluster +// is reachable. The second return value reports whether that URL is routable +// from outside the host cluster (true for NodePort/LoadBalancer/Ingress, false +// for plain ClusterIP exposition). +// +// hostServerIP is used as the address when the underlying Service is a +// NodePort. serverPort, when non-zero, overrides the port discovered from the +// Service. +func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, bool, error) { + key := types.NamespacedName{ + Name: ServiceName(cluster.Name), + Namespace: cluster.Namespace, + } + + var k3kService corev1.Service + if err := c.Get(ctx, key, &k3kService); err != nil { + return "", false, err + } + + ip := k3kService.Spec.ClusterIP + port := int32(httpsPort) + external := false + + if len(k3kService.Spec.Ports) == 0 { + logrus.Warn("No ports exposed by the cluster service.") + } + + switch k3kService.Spec.Type { + case corev1.ServiceTypeNodePort: + ip = hostServerIP + external = true + + if len(k3kService.Spec.Ports) > 0 { + port = k3kService.Spec.Ports[0].NodePort + } + case corev1.ServiceTypeLoadBalancer: + external = true + + if len(k3kService.Status.LoadBalancer.Ingress) > 0 { + ip = k3kService.Status.LoadBalancer.Ingress[0].IP + } else { + logrus.Warn("No ingress found in LoadBalancer service.") + } + + if len(k3kService.Spec.Ports) > 0 { + port = k3kService.Spec.Ports[0].Port + } + } + + if serverPort != 0 { + port = int32(serverPort) + } + + if !slices.Contains(cluster.Status.TLSSANs, ip) { + logrus.Warnf("IP %s not in tlsSANs.", ip) + + if len(cluster.Spec.TLSSANs) > 0 { + logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0]) + + ip = cluster.Spec.TLSSANs[0] + } else if len(cluster.Status.TLSSANs) > 0 { + logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0]) + + ip = cluster.Status.TLSSANs[0] + } else { + logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.") + } + } + + url := "https://" + ip + if port != httpsPort { + url = fmt.Sprintf("%s:%d", url, port) + } + + // if ingress is specified, use the ingress host + if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { + var k3kIngress networkingv1.Ingress + + ingressKey := types.NamespacedName{ + Name: IngressName(cluster.Name), + Namespace: cluster.Namespace, + } + + if err := c.Get(ctx, ingressKey, &k3kIngress); err != nil { + return "", external, err + } + + if len(k3kIngress.Spec.Rules) > 0 { + url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host) + external = true + } + } + + return url, external, nil +} diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 330ceefa..8ea304cb 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -22,7 +22,6 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" - "github.com/rancher/k3k/pkg/controller/cluster/agent" "github.com/rancher/k3k/pkg/controller/cluster/mounts" ) @@ -250,8 +249,10 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo }, }, } - // start the pod unprivileged in shared mode - if s.mode == agent.VirtualNodeMode { + // virtual mode runs an embedded kubelet inside the server pod and therefore + // requires Privileged. shared and hcp modes are agentless (no kubelet) and + // run unprivileged. + if s.mode == string(v1beta1.VirtualClusterMode) { podSpec.Containers[0].SecurityContext = &corev1.SecurityContext{ Privileged: ptr.To(true), } diff --git a/pkg/controller/cluster/server/template.go b/pkg/controller/cluster/server/template.go index daf83699..774fb985 100644 --- a/pkg/controller/cluster/server/template.go +++ b/pkg/controller/cluster/server/template.go @@ -19,7 +19,7 @@ safe_mode() { CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip) fi - if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ {{.K3K_MODE}} != "virtual" ]; then + if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" = "shared" ] || [ "{{.K3K_MODE}}" = "hcp" ]; then return fi @@ -116,7 +116,8 @@ configure_cgroups() { fi # only configure the cgroups if the runtime used is the default and the mode is virtual - if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then + # shared and hcp run agentless (no kubelet) and don't need cgroup overrides. + if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then return fi diff --git a/pkg/controller/cluster/status.go b/pkg/controller/cluster/status.go index 0a229f77..cff1024d 100644 --- a/pkg/controller/cluster/status.go +++ b/pkg/controller/cluster/status.go @@ -19,11 +19,12 @@ const ( ConditionReady = "Ready" // Condition Reasons - ReasonValidationFailed = "ValidationFailed" - ReasonProvisioning = "Provisioning" - ReasonProvisioned = "Provisioned" - ReasonProvisioningFailed = "ProvisioningFailed" - ReasonTerminating = "Terminating" + ReasonValidationFailed = "ValidationFailed" + ReasonProvisioning = "Provisioning" + ReasonProvisioned = "Provisioned" + ReasonProvisioningFailed = "ProvisioningFailed" + ReasonTerminating = "Terminating" + ReasonHCPNoExternalEndpoint = "HCPNoExternalEndpoint" ) func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.Cluster, reconcileErr error) { diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go index 44860314..86ecf9e3 100644 --- a/pkg/controller/kubeconfig/kubeconfig.go +++ b/pkg/controller/kubeconfig/kubeconfig.go @@ -3,18 +3,12 @@ package kubeconfig import ( "context" "crypto/x509" - "fmt" - "slices" "time" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/user" "sigs.k8s.io/controller-runtime/pkg/client" certutil "github.com/rancher/dynamiclistener/cert" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" @@ -60,7 +54,7 @@ func (k *KubeConfig) Generate(ctx context.Context, client client.Client, cluster return nil, err } - url, err := getURLFromService(ctx, client, cluster, hostServerIP, port) + url, _, err := server.ServerURL(ctx, client, cluster, hostServerIP, port) if err != nil { return nil, err } @@ -93,84 +87,3 @@ func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi return config } -func getURLFromService(ctx context.Context, client client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, error) { - // get the server service to extract the right IP - key := types.NamespacedName{ - Name: server.ServiceName(cluster.Name), - Namespace: cluster.Namespace, - } - - var k3kService corev1.Service - if err := client.Get(ctx, key, &k3kService); err != nil { - return "", err - } - - ip := k3kService.Spec.ClusterIP - port := int32(443) - - if len(k3kService.Spec.Ports) == 0 { - logrus.Warn("No ports exposed by the cluster service.") - } - - switch k3kService.Spec.Type { - case corev1.ServiceTypeNodePort: - ip = hostServerIP - - if len(k3kService.Spec.Ports) > 0 { - port = k3kService.Spec.Ports[0].NodePort - } - case corev1.ServiceTypeLoadBalancer: - if len(k3kService.Status.LoadBalancer.Ingress) > 0 { - ip = k3kService.Status.LoadBalancer.Ingress[0].IP - } else { - logrus.Warn("No ingress found in LoadBalancer service.") - } - - if len(k3kService.Spec.Ports) > 0 { - port = k3kService.Spec.Ports[0].Port - } - } - - if serverPort != 0 { - port = int32(serverPort) - } - - if !slices.Contains(cluster.Status.TLSSANs, ip) { - logrus.Warnf("IP %s not in tlsSANs.", ip) - - if len(cluster.Spec.TLSSANs) > 0 { - logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0]) - - ip = cluster.Spec.TLSSANs[0] - } else if len(cluster.Status.TLSSANs) > 0 { - logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0]) - - ip = cluster.Status.TLSSANs[0] - } else { - logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.") - } - } - - url := "https://" + ip - if port != 443 { - url = fmt.Sprintf("%s:%d", url, port) - } - - // if ingress is specified, use the ingress host - if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { - var k3kIngress networkingv1.Ingress - - ingressKey := types.NamespacedName{ - Name: server.IngressName(cluster.Name), - Namespace: cluster.Namespace, - } - - if err := client.Get(ctx, ingressKey, &k3kIngress); err != nil { - return "", err - } - - url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host) - } - - return url, nil -} diff --git a/scripts/generate b/scripts/generate index 820d958f..5b618ca4 100755 --- a/scripts/generate +++ b/scripts/generate @@ -15,5 +15,5 @@ go run sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSIO # add the 'helm.sh/resource-policy: keep' annotation to the CRDs for f in ./charts/k3k/templates/crds/*.yaml; do echo "Annotating $f" - yq -c -i '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$f" + #yq -c -i '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$f" done diff --git a/tests/e2e/common_test.go b/tests/e2e/common_test.go index 67668724..ed44c396 100644 --- a/tests/e2e/common_test.go +++ b/tests/e2e/common_test.go @@ -288,7 +288,8 @@ func (c *VirtualCluster) NewNginxPod(namespace string) (*corev1.Pod, string) { By(fmt.Sprintf("Nginx Pod is running (%s/%s)", nginxPod.Namespace, nginxPod.Name)) - // only check the pod on the host cluster if the mode is shared mode + // only check the pod on the host cluster if the mode is shared mode. + // hcp is agentless and BYO-node, so no host-side pod mirror exists. if c.Cluster.Spec.Mode != v1beta1.SharedClusterMode { return nginxPod, "" } From bda74aa14c7aed552cc209b2e64a1261518d4eae Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 25 May 2026 13:41:29 +0200 Subject: [PATCH 02/36] fix merge --- pkg/controller/cluster/server/config.go | 44 ++++++++++++++----------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index 25aafd8f..14978953 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -22,6 +22,7 @@ type serverConfig struct { DisableAgent bool `yaml:"disable-agent,omitempty"` Disable []string `yaml:"disable,omitempty"` EgressSelectorMode string `yaml:"egress-selector-mode,omitempty"` + KubeApiServerArg []string `yaml:"kube-apiserver-arg,omitempty"` Server string `yaml:"server,omitempty"` ServiceCIDR string `yaml:"service-cidr,omitempty"` TLSSAN []string `yaml:"tls-san,omitempty"` @@ -77,10 +78,30 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok } // shared and hcp modes both run K3s with --disable-agent (agentless server). - // hcp additionally relies on this to satisfy the PRD requirement that the - // control plane never runs a kubelet and is not enumerated as a node. - if cluster.Spec.Mode != v1beta1.VirtualClusterMode { - opts = opts + "disable-agent: true\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage\n" + switch cluster.Spec.Mode { + case v1beta1.SharedClusterMode: + serverConfig.DisableAgent = true + serverConfig.EgressSelectorMode = "disabled" + serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"} + case v1beta1.HCPClusterMode: + serverConfig.DisableAgent = true + serverConfig.EgressSelectorMode = "cluster" + serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"} + // Disable it so K3k can own that Endpoints object and point + // it at the externally-reachable host:port (NodePort / LB / Ingress). + serverConfig.KubeApiServerArg = append(serverConfig.KubeApiServerArg, "endpoint-reconciler-type=none") + case v1beta1.VirtualClusterMode: + // no extra config for virtual mode + } + + // In hcp mode the apiserver pod IP is unreachable from external worker + // nodes, so the kube-apiserver's default lease-based endpoint reconciler + // would publish a broken default/kubernetes Endpoints (advertise-address + + // secure-port). Disable it so K3k can own that Endpoints object and point + // it at the externally-reachable host:port (NodePort / LB / Ingress). + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + serverConfig.KubeApiServerArg = append(serverConfig.KubeApiServerArg, "endpoint-reconciler-type=none") + serverConfig.EgressSelectorMode = "cluster" } // In shared mode workloads run on the host cluster, so the apiserver pod @@ -107,21 +128,6 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok // non-hostNet entries, so every pod IP and every node port // is permitted. No race, no per-port allowlist. This is // what we want for a managed control plane. - switch cluster.Spec.Mode { - case v1beta1.SharedClusterMode: - opts = opts + "egress-selector-mode: disabled\n" - case v1beta1.HCPClusterMode: - opts = opts + "egress-selector-mode: cluster\n" - } - - // In hcp mode the apiserver pod IP is unreachable from external worker - // nodes, so the kube-apiserver's default lease-based endpoint reconciler - // would publish a broken default/kubernetes Endpoints (advertise-address + - // secure-port). Disable it so K3k can own that Endpoints object and point - // it at the externally-reachable host:port (NodePort / LB / Ingress). - if cluster.Spec.Mode == v1beta1.HCPClusterMode { - opts = opts + "kube-apiserver-arg:\n- endpoint-reconciler-type=none\n" - } return serverConfig } From e0dbbb9046f5d19ce1282d1013ab0691af29ad42 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 3 Jun 2026 14:31:21 +0200 Subject: [PATCH 03/36] test --- .github/workflows/test.yaml | 86 +++++++++++++++++++++++++ Makefile | 2 +- pkg/controller/cluster/server/config.go | 1 - 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 23d9d5f4..d8367de7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,6 +14,92 @@ env: KUBERNETES_VERSION: v1.35.3 jobs: + hcp-test: + runs-on: ubuntu-latest + + steps: + - name: Install Virtualization Dependencies + run: | + sudo apt-get update + sudo apt-get install -y qemu-kvm qemu-utils cloud-image-utils wget + sudo usermod -aG kvm $USER + + kvm-ok + + - name: Download Base Cloud Image + run: | + wget -q https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img + + - name: Generate SSH Key and Cloud-Init Seed + run: | + ssh-keygen -t rsa -b 4096 -f ./id_rsa -N "" + + # Define the cloud-init config to authorize our new key + cat < user-data + #cloud-config + users: + - name: ubuntu + ssh_authorized_keys: + - $(cat ./id_rsa.pub) + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + shell: /bin/bash + EOF + + # Bake the configuration into a reusable metadata ISO + cloud-localds seed.img user-data + + # 4. Provision independent overlay disks for Worker + - name: Create Worker Disks + run: | + qemu-img create -f qcow2 -b focal-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G + + # 5. Boot both VMs headlessly in the background with unique MACs and forwarded SSH ports + - name: Launch Worker VM + run: | + # Launch Worker 1 (SSH forwarded to Host Port 2222) + sudo qemu-system-x86_64 \ + -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ + -drive file=worker-1.qcow2,if=virtio \ + -drive file=seed.img,format=raw,if=virtio \ + -net nic,model=virtio,macaddr=52:54:00:12:34:56 \ + -net user,hostfwd=tcp::2222-:22 \ + & + + # 6. Block the script until both VMs are completely booted and listening on SSH + - name: Wait for SSH Availability + run: | + echo "Waiting for Worker (Port 2222) to respond..." + timeout 120s bash -c ' + until ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done + ' + + echo "VM is completely up and accessible!" + + + # # 7. Extract token and trigger the manual join command via SSH + # - name: Join Worker to K3k Control Plane + # run: | + # # Fetch the registration secret from your K3k virtual cluster context + # # (Update namespace/secret names to match your precise K3k CRD setup) + # K3K_TOKEN=$(kubectl get secret -n k3k-system-cluster -o jsonpath='{.data.token}' | base64 -d) + + # echo "Registering Worker 1..." + # ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + # "curl -sfL https://get.k3s.io | K3S_URL='https://10.0.2.2:6443' K3S_TOKEN='${K3K_TOKEN}' sh -s - agent" + + # # 8. Assert that both nodes successfully registered and transitioned to a Ready status + # - name: Verify Cluster Nodes + # run: | + # echo "Monitoring K3k Virtual Cluster for Node registration..." + # timeout 180s bash -c ' + # until [ $(kubectl --kubeconfig=k3k-cluster.kubeconfig get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 2 ]; do + # echo "Waiting for both worker nodes to show Ready..." + # kubectl --kubeconfig=k3k-cluster.kubeconfig get nodes || true + # sleep 5 + # done + # ' + # echo "E2E Success: Both QEMU workers successfully attached to the Hosted Control Plane!" + tests: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 0b6dd6f9..64a24330 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ package: package-k3k package-k3k-kubelet ## Package the k3k and k3k-kubelet Dock .PHONY: package-% package-%: - docker buildx build --platform linux/amd64,linux/arm64 -f package/Dockerfile.$* \ + docker build -f package/Dockerfile.$* \ -t $(REPO)/$*:$(VERSION) \ -t $(REPO)/$*:latest \ -t $(REPO)/$*:dev . diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index 14978953..428b808c 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -86,7 +86,6 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok case v1beta1.HCPClusterMode: serverConfig.DisableAgent = true serverConfig.EgressSelectorMode = "cluster" - serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"} // Disable it so K3k can own that Endpoints object and point // it at the externally-reachable host:port (NodePort / LB / Ingress). serverConfig.KubeApiServerArg = append(serverConfig.KubeApiServerArg, "endpoint-reconciler-type=none") From ab491fd24b76fa10736461853e1674c4cb6c4dc3 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 3 Jun 2026 16:43:58 +0200 Subject: [PATCH 04/36] setup k3k --- .github/workflows/test.yaml | 108 ++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d8367de7..eb573192 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,12 +12,120 @@ permissions: env: KUBERNETES_VERSION: v1.35.3 + HELM_VERSION: v4.1.3 + HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700 + K3D_VERSION: v5.8.3 + K3D_CHECKSUM_AMD64: dbaa79a76ace7f4ca230a1ff41dc7d8a5036a8ad0309e9c54f9bf3836dbe853e jobs: hcp-test: runs-on: ubuntu-latest steps: + - name: Install helm + env: + FILENAME: helm.tar.gz + run: | + curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz + echo "${{ env.HELM_CHECKSUM_AMD64 }} ${{ env.FILENAME }}" | sha256sum --check + tar -xvzf ${{ env.FILENAME }} linux-amd64/helm + sudo install -m 755 linux-amd64/helm /usr/local/bin/helm + + rm -fr "${{ env.FILENAME }}" linux-amd64/helm + + - name: Install hydrophone + run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0 + + - name: Install k3d + run: | + curl -sSfL -o k3d "https://github.com/k3d-io/k3d/releases/download/${{ env.K3D_VERSION }}/k3d-linux-amd64" + echo "${{ env.K3D_CHECKSUM_AMD64 }} k3d" | sha256sum --check + sudo install -m 755 k3d /usr/local/bin/k3d + + rm -f k3d + + - name: Install kubectl + run: | + curl -LO "https://dl.k8s.io/release/${{ env.KUBERNETES_VERSION }}/bin/linux/amd64/kubectl" + curl -LO "https://dl.k8s.io/release/${{ env.KUBERNETES_VERSION }}/bin/linux/amd64/kubectl.sha256" + echo "$(cat kubectl.sha256) kubectl" | sha256sum --check + + - name: Setup Kubernetes (k3d) + env: + REPO_NAME: k3k-registry + REPO_PORT: 12345 + run: | + echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts + + k3d registry create ${REPO_NAME} --port ${REPO_PORT} + + k3d cluster create k3k --servers 2 \ + --image rancher/k3s:${{ env.KUBERNETES_VERSION }}-k3s1 \ + -p "30000-30010:30000-30010@server:0" \ + --registry-use k3d-${REPO_NAME}:${REPO_PORT} + + kubectl cluster-info + kubectl get nodes + + - name: Setup K3k (from source) + env: + REPO: k3k-registry:12345 + run: | + echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts + + make build + make package + make push + + # add k3kcli to $PATH + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + + VERSION=$(make version) + k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose + k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose + + make install + + - name: Wait for K3k controller + run: | + echo "Wait for K3k controller deployment to be available" + kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m + + - name: Check k3kcli + run: k3kcli -v + + - name: Create virtual cluster + run: | + kubectl create namespace k3k-mycluster + + cat < Date: Wed, 3 Jun 2026 19:04:17 +0200 Subject: [PATCH 05/36] missing checkout --- .github/workflows/test.yaml | 50 +++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eb573192..1846e638 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,6 +22,16 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version-file: go.mod + - name: Install helm env: FILENAME: helm.tar.gz @@ -184,29 +194,27 @@ jobs: echo "VM is completely up and accessible!" - # # 7. Extract token and trigger the manual join command via SSH - # - name: Join Worker to K3k Control Plane - # run: | - # # Fetch the registration secret from your K3k virtual cluster context - # # (Update namespace/secret names to match your precise K3k CRD setup) - # K3K_TOKEN=$(kubectl get secret -n k3k-system-cluster -o jsonpath='{.data.token}' | base64 -d) + - name: Join Worker to K3k Control Plane + run: | + echo "Getting cluster token" + K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) - # echo "Registering Worker 1..." - # ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - # "curl -sfL https://get.k3s.io | K3S_URL='https://10.0.2.2:6443' K3S_TOKEN='${K3K_TOKEN}' sh -s - agent" + echo "Registering Worker..." + ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -sfL https://get.k3s.io | K3S_URL='https://10.0.2.2:6443' K3S_TOKEN='${K3S_TOKEN}' sh -s - agent" - # # 8. Assert that both nodes successfully registered and transitioned to a Ready status - # - name: Verify Cluster Nodes - # run: | - # echo "Monitoring K3k Virtual Cluster for Node registration..." - # timeout 180s bash -c ' - # until [ $(kubectl --kubeconfig=k3k-cluster.kubeconfig get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 2 ]; do - # echo "Waiting for both worker nodes to show Ready..." - # kubectl --kubeconfig=k3k-cluster.kubeconfig get nodes || true - # sleep 5 - # done - # ' - # echo "E2E Success: Both QEMU workers successfully attached to the Hosted Control Plane!" + # 8. Assert that both nodes successfully registered and transitioned to a Ready status + - name: Verify Cluster Nodes + run: | + echo "Monitoring K3k Virtual Cluster for Node registration..." + timeout 180s bash -c ' + until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 2 ]; do + echo "Waiting for both worker nodes to show Ready..." + kubectl get nodes || true + sleep 5 + done + ' + echo "E2E Success: Both QEMU workers successfully attached to the Hosted Control Plane!" tests: runs-on: ubuntu-latest From 898b2f34f0614ebe370e3472ab110c529b989b19 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 3 Jun 2026 19:34:47 +0200 Subject: [PATCH 06/36] removed k3d --- .github/workflows/test.yaml | 161 +++++++++++++++------- pkg/controller/cluster/cluster.go | 2 +- pkg/controller/cluster/hcp.go | 135 +++++++++++++----- pkg/controller/cluster/hcp_test.go | 149 ++++++++++++++++++-- pkg/controller/cluster/server/endpoint.go | 5 +- pkg/controller/kubeconfig/kubeconfig.go | 1 - 6 files changed, 349 insertions(+), 104 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1846e638..b9a288b9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,8 +14,6 @@ env: KUBERNETES_VERSION: v1.35.3 HELM_VERSION: v4.1.3 HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700 - K3D_VERSION: v5.8.3 - K3D_CHECKSUM_AMD64: dbaa79a76ace7f4ca230a1ff41dc7d8a5036a8ad0309e9c54f9bf3836dbe853e jobs: hcp-test: @@ -43,59 +41,31 @@ jobs: rm -fr "${{ env.FILENAME }}" linux-amd64/helm - - name: Install hydrophone - run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0 - - - name: Install k3d - run: | - curl -sSfL -o k3d "https://github.com/k3d-io/k3d/releases/download/${{ env.K3D_VERSION }}/k3d-linux-amd64" - echo "${{ env.K3D_CHECKSUM_AMD64 }} k3d" | sha256sum --check - sudo install -m 755 k3d /usr/local/bin/k3d - - rm -f k3d - - - name: Install kubectl - run: | - curl -LO "https://dl.k8s.io/release/${{ env.KUBERNETES_VERSION }}/bin/linux/amd64/kubectl" - curl -LO "https://dl.k8s.io/release/${{ env.KUBERNETES_VERSION }}/bin/linux/amd64/kubectl.sha256" - echo "$(cat kubectl.sha256) kubectl" | sha256sum --check - - - name: Setup Kubernetes (k3d) + - name: Install k3s env: - REPO_NAME: k3k-registry - REPO_PORT: 12345 + K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 run: | - echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV - k3d registry create ${REPO_NAME} --port ${REPO_PORT} - - k3d cluster create k3k --servers 2 \ - --image rancher/k3s:${{ env.KUBERNETES_VERSION }}-k3s1 \ - -p "30000-30010:30000-30010@server:0" \ - --registry-use k3d-${REPO_NAME}:${REPO_PORT} - kubectl cluster-info kubectl get nodes - name: Setup K3k (from source) - env: - REPO: k3k-registry:12345 run: | - echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts + export REPO=ttl.sh/$(uuidgen) + export VERSION=1h make build make package make push + make install # add k3kcli to $PATH echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - VERSION=$(make version) - k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose - k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose - - make install - - name: Wait for K3k controller run: | echo "Wait for K3k controller deployment to be available" @@ -116,7 +86,6 @@ jobs: namespace: k3k-mycluster spec: mode: hcp - mirrorHostNodes: true tlsSANs: - "127.0.0.1" - "10.0.2.2" @@ -131,11 +100,36 @@ jobs: k3kcli kubeconfig generate --name mycluster export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml - + kubectl cluster-info kubectl get nodes kubectl get pods -A + - name: Wait for cluster to be ready + run: | + echo "Waiting for cluster to reach Ready phase..." + + timeout 300s bash -c ' + until kubectl get cluster -n k3k-mycluster mycluster -o jsonpath="{.status.phase}" | grep -q "Ready"; do + PHASE=$(kubectl get cluster -n k3k-mycluster mycluster -o jsonpath="{.status.phase}") + echo "Current phase: ${PHASE}" + sleep 5 + done + ' + + echo "✓ Cluster is ready!" + + echo "Verifying NodePort service is accessible from host..." + + if ! curl -k --max-time 5 https://127.0.0.1:30001/readyz; then + echo "ERROR: Cannot reach NodePort service from host!" + echo "Service details:" + kubectl get svc -n k3k-mycluster -l cluster=mycluster,role=server -o yaml + exit 1 + fi + + echo "✓ NodePort service is accessible" + - name: Install Virtualization Dependencies run: | sudo apt-get update @@ -146,7 +140,7 @@ jobs: - name: Download Base Cloud Image run: | - wget -q https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img + wget -q https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img - name: Generate SSH Key and Cloud-Init Seed run: | @@ -169,7 +163,7 @@ jobs: # 4. Provision independent overlay disks for Worker - name: Create Worker Disks run: | - qemu-img create -f qcow2 -b focal-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G + qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G # 5. Boot both VMs headlessly in the background with unique MACs and forwarded SSH ports - name: Launch Worker VM @@ -196,25 +190,94 @@ jobs: - name: Join Worker to K3k Control Plane run: | - echo "Getting cluster token" K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) - - echo "Registering Worker..." + + echo "Testing connectivity from VM to K3k API server..." ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -sfL https://get.k3s.io | K3S_URL='https://10.0.2.2:6443' K3S_TOKEN='${K3S_TOKEN}' sh -s - agent" + "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'VM connectivity test failed'" + + echo "Registering Worker..." + set +e # Don't exit on error, we want to collect logs + ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 bash -s </dev/null | grep -c "Ready") -eq 2 ]; do + until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 1 ]; do echo "Waiting for both worker nodes to show Ready..." kubectl get nodes || true sleep 5 done ' - echo "E2E Success: Both QEMU workers successfully attached to the Hosted Control Plane!" + + - name: Collect logs + if: always() + run: | + journalctl -u k3s -o cat --no-pager > /tmp/k3s.log + kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log + + - name: Archive K3s logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3s-hcp-logs + path: /tmp/k3s.log + + - name: Archive K3k logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3k-hcp-logs + path: /tmp/k3k.log tests: runs-on: ubuntu-latest diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index dad6447e..91cd7ce5 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -458,7 +458,7 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus return err } - if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil { + if err := c.ensureHCPKubernetesEndpointSlice(ctx, cluster); err != nil { return err } } diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go index 8d82cd8b..dff7fab8 100644 --- a/pkg/controller/cluster/hcp.go +++ b/pkg/controller/cluster/hcp.go @@ -7,23 +7,18 @@ import ( "net/url" "strconv" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller/cluster/server" ) -// endpointSliceSkipMirrorLabel is the upstream label that opts an Endpoints -// object out of the kube-controller-manager EndpointSlice mirroring controller. -// The kube-apiserver normally sets it on default/kubernetes (because it -// manages EndpointSlices itself); in HCP mode we want the mirror controller -// to handle slices, so we strip the label. -const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror" - // ensureHCPRegistration computes the K3s installer command external nodes can // run to join an HCP-mode cluster and stores it on cluster.Status.HCPRegistration. // @@ -34,7 +29,7 @@ const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror" func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error { log := ctrl.LoggerFrom(ctx) - url, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0) + url, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0) if err != nil { return err } @@ -76,22 +71,55 @@ func hcpRegistrationCommand(version, serverURL, token string) string { version, serverURL, token) } -// ensureHCPKubernetesEndpoints maintains the default/kubernetes Service -// Endpoints inside the virtual cluster, pointing them at the externally +// selectNonLoopbackSAN returns the first non-loopback address from the +// cluster's TLS SANs, preferring spec.TLSSANs then falling back to status.TLSSANs. +// Returns empty string if no non-loopback address is found. +func selectNonLoopbackSAN(cluster *v1beta1.Cluster) string { + // Try spec.TLSSANs first (user-provided values) + for _, san := range cluster.Spec.TLSSANs { + if ip := net.ParseIP(san); ip != nil && ip.IsLoopback() { + continue + } + + if san == "localhost" { + continue + } + + return san // Found a non-loopback address + } + + // Fall back to status.TLSSANs (computed values) + for _, san := range cluster.Status.TLSSANs { + if ip := net.ParseIP(san); ip != nil && ip.IsLoopback() { + continue + } + + if san == "localhost" { + continue + } + + return san + } + + return "" +} + +// ensureHCPKubernetesEndpointSlice maintains the default/kubernetes Service +// EndpointSlice inside the virtual cluster, pointing it at the externally // reachable host:port (NodePort / LoadBalancer / Ingress) so that pods // scheduled on external worker nodes can reach the in-cluster apiserver // ClusterIP. // // Background: the kube-apiserver normally reconciles default/kubernetes -// Endpoints to its own --advertise-address:--secure-port (the host-cluster +// EndpointSlice to its own --advertise-address:--secure-port (the host-cluster // pod IP and 6443). External worker nodes have no route to the host-cluster // pod CIDR, so kube-proxy DNAT to that endpoint fails. We disable the // apiserver reconciler in HCP mode (see serverOptions) and own this -// Endpoints object instead. -func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error { +// EndpointSlice object instead. +func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context, cluster *v1beta1.Cluster) error { log := ctrl.LoggerFrom(ctx) - rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0) + rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0) if err != nil { return err } @@ -112,45 +140,65 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cl return err } + var addressType discoveryv1.AddressType + + if ip := net.ParseIP(addr.IP); ip != nil { + if ip.To4() != nil { + addressType = discoveryv1.AddressTypeIPv4 + } else { + addressType = discoveryv1.AddressTypeIPv6 + } + } else { + return fmt.Errorf("invalid IP address %q", addr.IP) + } + virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace) if err != nil { return fmt.Errorf("creating virtual cluster client: %w", err) } - endpoints := &corev1.Endpoints{ + endpointSlice := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "kubernetes", Namespace: metav1.NamespaceDefault, }, } - _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error { - // Allow EndpointSlice mirroring; the apiserver may have set - // skip-mirror=true before we disabled its endpoint reconciler. - if endpoints.Labels != nil { - delete(endpoints.Labels, endpointSliceSkipMirrorLabel) + _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpointSlice, func() error { + // Ensure the service-name label is set + if endpointSlice.Labels == nil { + endpointSlice.Labels = make(map[string]string) } - endpoints.Subsets = []corev1.EndpointSubset{ + endpointSlice.Labels[discoveryv1.LabelServiceName] = "kubernetes" + endpointSlice.AddressType = addressType + + endpoint := discoveryv1.Endpoint{ + Addresses: []string{addr.IP}, + } + + if addr.Hostname != "" { + endpoint.Hostname = &addr.Hostname + } + + endpointSlice.Endpoints = []discoveryv1.Endpoint{endpoint} + portName := "https" + + endpointSlice.Ports = []discoveryv1.EndpointPort{ { - Addresses: []corev1.EndpointAddress{addr}, - Ports: []corev1.EndpointPort{ - { - Name: "https", - Port: port, - Protocol: corev1.ProtocolTCP, - }, - }, + Name: &portName, + Port: &port, + Protocol: new(corev1.ProtocolTCP), }, } return nil }) if err != nil { - return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err) + return fmt.Errorf("upserting default/kubernetes endpointslice in virtual cluster: %w", err) } - log.V(1).Info("HCP kubernetes endpoints reconciled", + log.V(1).Info("HCP kubernetes endpointslice reconciled", "address", addr.IP, "hostname", addr.Hostname, "port", port) return nil @@ -172,6 +220,7 @@ func parseHCPHostPort(rawURL string) (string, int32, error) { portStr := u.Port() var port int32 = 443 + if portStr != "" { p, err := strconv.Atoi(portStr) if err != nil { @@ -194,6 +243,10 @@ func parseHCPHostPort(rawURL string) (string, int32, error) { // human-readable. func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is a loopback address and cannot be used", host) + } + return corev1.EndpointAddress{IP: host}, nil } @@ -202,15 +255,21 @@ func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err) } + var filteredIPs []net.IP + for _, ip := range ips { - if v4 := ip.To4(); v4 != nil { - return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil + if !ip.IsLoopback() { + filteredIPs = append(filteredIPs, ip) } } - if len(ips) == 0 { - return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no IPs", host) + if len(filteredIPs) == 0 { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no non-loopback IPs", host) } - return corev1.EndpointAddress{IP: ips[0].String(), Hostname: host}, nil + if v4 := filteredIPs[0].To4(); v4 != nil { + return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil + } + + return corev1.EndpointAddress{IP: filteredIPs[0].String(), Hostname: host}, nil } diff --git a/pkg/controller/cluster/hcp_test.go b/pkg/controller/cluster/hcp_test.go index a7edb8a9..6587b0aa 100644 --- a/pkg/controller/cluster/hcp_test.go +++ b/pkg/controller/cluster/hcp_test.go @@ -7,13 +7,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" @@ -180,18 +179,142 @@ func Test_parseHCPHostPort(t *testing.T) { } } -func Test_hcpEndpointAddress(t *testing.T) { - t.Run("ipv4 literal is passed through", func(t *testing.T) { - got, err := hcpEndpointAddress("10.144.101.195") - require.NoError(t, err) - assert.Equal(t, "10.144.101.195", got.IP) - assert.Empty(t, got.Hostname) - }) +func Test_selectNonLoopbackSAN(t *testing.T) { + tests := []struct { + name string + specSANs []string + statusSANs []string + want string + }{ + { + name: "spec with loopback first, external second", + specSANs: []string{"127.0.0.1", "10.0.0.100"}, + statusSANs: []string{}, + want: "10.0.0.100", + }, + { + name: "spec with external first", + specSANs: []string{"10.0.0.100", "127.0.0.1"}, + statusSANs: []string{}, + want: "10.0.0.100", + }, + { + name: "spec empty, status with loopback first, external second", + specSANs: []string{}, + statusSANs: []string{"127.0.0.1", "10.43.0.50"}, + want: "10.43.0.50", + }, + { + name: "spec with only loopback", + specSANs: []string{"127.0.0.1", "::1"}, + statusSANs: []string{}, + want: "", + }, + { + name: "spec with localhost hostname", + specSANs: []string{"localhost", "example.com"}, + statusSANs: []string{}, + want: "example.com", + }, + { + name: "spec with external hostname", + specSANs: []string{"hcp.example.com"}, + statusSANs: []string{}, + want: "hcp.example.com", + }, + { + name: "empty spec and status", + specSANs: []string{}, + statusSANs: []string{}, + want: "", + }, + { + name: "ipv6 loopback filtered", + specSANs: []string{"::1", "2001:db8::1"}, + statusSANs: []string{}, + want: "2001:db8::1", + }, + } - t.Run("unresolvable hostname errors", func(t *testing.T) { - _, err := hcpEndpointAddress("definitely-not-a-real-host.invalid") - require.Error(t, err) - }) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cluster := &v1beta1.Cluster{ + Spec: v1beta1.ClusterSpec{ + TLSSANs: tt.specSANs, + }, + Status: v1beta1.ClusterStatus{ + TLSSANs: tt.statusSANs, + }, + } + got := selectNonLoopbackSAN(cluster) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_hcpEndpointAddress(t *testing.T) { + tests := []struct { + name string + input string + wantIP string + wantHostname string + wantErr bool + }{ + { + name: "ipv4 literal is passed through", + input: "10.144.101.195", + wantIP: "10.144.101.195", + wantHostname: "", + wantErr: false, + }, + { + name: "unresolvable hostname errors", + input: "definitely-not-a-real-host.invalid", + wantErr: true, + }, + { + name: "ipv4 loopback literal is rejected", + input: "127.0.0.1", + wantErr: true, + }, + { + name: "ipv6 loopback literal is rejected", + input: "::1", + wantErr: true, + }, + { + name: "ipv4 loopback in range is rejected", + input: "127.0.0.100", + wantErr: true, + }, + { + name: "valid ipv6 literal passes through", + input: "2001:db8::1", + wantIP: "2001:db8::1", + wantHostname: "", + wantErr: false, + }, + { + name: "localhost hostname filters loopbacks", + input: "localhost", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := hcpEndpointAddress(tt.input) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + + assert.Equal(t, tt.wantIP, got.IP) + assert.Equal(t, tt.wantHostname, got.Hostname) + }) + } } // Compile-time assertion: every reused exported name from the controller diff --git a/pkg/controller/cluster/server/endpoint.go b/pkg/controller/cluster/server/endpoint.go index 13066304..23962630 100644 --- a/pkg/controller/cluster/server/endpoint.go +++ b/pkg/controller/cluster/server/endpoint.go @@ -6,11 +6,12 @@ import ( "slices" "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" ) diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go index 86ecf9e3..9bb4cf8b 100644 --- a/pkg/controller/kubeconfig/kubeconfig.go +++ b/pkg/controller/kubeconfig/kubeconfig.go @@ -86,4 +86,3 @@ func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi return config } - From cff7141af74984ab978905509ba124beb9034d10 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 10 Jun 2026 11:18:53 +0200 Subject: [PATCH 07/36] removed hcpRegitration command --- .../k3k/templates/crds/k3k.io_clusters.yaml | 7 --- docs/cli/k3kcli.adoc | 2 +- docs/cli/k3kcli_cluster_create.md | 2 +- docs/crds/crds.adoc | 8 +-- docs/crds/crds.md | 6 +- pkg/apis/k3k.io/v1beta1/types.go | 8 --- pkg/controller/cluster/hcp.go | 24 +------ pkg/controller/cluster/hcp_test.go | 63 ------------------- 8 files changed, 10 insertions(+), 110 deletions(-) diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index 9eee4c0e..ca528890 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -3173,13 +3173,6 @@ spec: - type type: object type: array - hcpRegistration: - description: |- - HCPRegistration is a copy-pasteable K3s installer command that external - (BYO) nodes can run to register against an HCP-mode cluster. - Only populated when Mode is "hcp" and an externally-routable endpoint - (NodePort, LoadBalancer or Ingress) is configured. - type: string hostVersion: description: HostVersion is the Kubernetes version of the host node. type: string diff --git a/docs/cli/k3kcli.adoc b/docs/cli/k3kcli.adoc index b6fbebf5..e31ed8c7 100644 --- a/docs/cli/k3kcli.adoc +++ b/docs/cli/k3kcli.adoc @@ -54,7 +54,7 @@ k3kcli cluster create [command options] NAME --kubeconfig-server string override the kubeconfig server host --labels stringArray Labels to add to the cluster object (e.g. key=value) --mirror-host-nodes Mirror Host Cluster Nodes - --mode string k3k mode type (shared, virtual) (default "shared") + --mode string k3k mode type (shared, virtual, hcp) (default "shared") -n, --namespace string namespace of the k3k cluster --persistence-type string persistence mode for the nodes (dynamic, ephemeral) (default "dynamic") --policy string The policy to create the cluster in diff --git a/docs/cli/k3kcli_cluster_create.md b/docs/cli/k3kcli_cluster_create.md index 7afccb21..dc487f24 100644 --- a/docs/cli/k3kcli_cluster_create.md +++ b/docs/cli/k3kcli_cluster_create.md @@ -25,7 +25,7 @@ k3kcli cluster create [command options] NAME --kubeconfig-server string override the kubeconfig server host --labels stringArray Labels to add to the cluster object (e.g. key=value) --mirror-host-nodes Mirror Host Cluster Nodes - --mode string k3k mode type (shared, virtual) (default "shared") + --mode string k3k mode type (shared, virtual, hcp) (default "shared") -n, --namespace string namespace of the k3k cluster --persistence-type string persistence mode for the nodes (dynamic, ephemeral) (default "dynamic") --policy string The policy to create the cluster in diff --git a/docs/crds/crds.adoc b/docs/crds/crds.adoc index 6d01db49..cd57d5d8 100644 --- a/docs/crds/crds.adoc +++ b/docs/crds/crds.adoc @@ -134,7 +134,7 @@ _Underlying type:_ _string_ ClusterMode is the possible provisioning mode of a Cluster. _Validation:_ -- Enum: [shared virtual] +- Enum: [shared virtual hcp] _Appears In:_ @@ -177,8 +177,8 @@ _Appears In:_ | *`version`* __string__ | Version is the K3s version to use for the virtual nodes. + It should follow the K3s versioning convention (e.g., v1.28.2-k3s1). + If not specified, the Kubernetes version of the host node will be used. + | | -| *`mode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | Mode specifies the cluster provisioning mode: "shared" or "virtual". + -Defaults to "shared". This field is immutable. + | shared | Enum: [shared virtual] + +| *`mode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp". + +Defaults to "shared". This field is immutable. + | shared | Enum: [shared virtual hcp] + | *`servers`* __integer__ | Servers specifies the number of K3s pods to run in server (control plane) mode. + Must be at least 1. Defaults to 1. + | 1 | @@ -790,7 +790,7 @@ to set defaults and constraints (min/max) + | | This includes both node affinity and pod affinity/anti-affinity rules. + | | | *`defaultAgentAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | DefaultAgentAffinity specifies the affinity rules applied to agent pods of all clusters in the target Namespace. + This includes both node affinity and pod affinity/anti-affinity rules. + | | -| *`allowedMode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". + | shared | Enum: [shared virtual] + +| *`allowedMode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". + | shared | Enum: [shared virtual hcp] + | *`disableNetworkPolicy`* __boolean__ | DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. + | | | *`podSecurityAdmissionLevel`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-podsecurityadmissionlevel[$$PodSecurityAdmissionLevel$$]__ | PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. + | | Enum: [privileged baseline restricted] + diff --git a/docs/crds/crds.md b/docs/crds/crds.md index 35bd7a27..a43bd48b 100644 --- a/docs/crds/crds.md +++ b/docs/crds/crds.md @@ -103,7 +103,7 @@ _Underlying type:_ _string_ ClusterMode is the possible provisioning mode of a Cluster. _Validation:_ -- Enum: [shared virtual] +- Enum: [shared virtual hcp] _Appears in:_ - [ClusterSpec](#clusterspec) @@ -138,7 +138,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `version` _string_ | Version is the K3s version to use for the virtual nodes.
It should follow the K3s versioning convention (e.g., v1.28.2-k3s1).
If not specified, the Kubernetes version of the host node will be used. | | | -| `mode` _[ClusterMode](#clustermode)_ | Mode specifies the cluster provisioning mode: "shared" or "virtual".
Defaults to "shared". This field is immutable. | shared | Enum: [shared virtual]
| +| `mode` _[ClusterMode](#clustermode)_ | Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp".
Defaults to "shared". This field is immutable. | shared | Enum: [shared virtual hcp]
| | `servers` _integer_ | Servers specifies the number of K3s pods to run in server (control plane) mode.
Must be at least 1. Defaults to 1. | 1 | | | `agents` _integer_ | Agents specifies the number of K3s pods to run in agent (worker) mode.
Must be 0 or greater. Defaults to 0.
This field is ignored in "shared" mode. | 0 | | | `clusterCIDR` _string_ | ClusterCIDR is the CIDR range for pod IPs.
Defaults to 10.42.0.0/16 in shared mode and 10.52.0.0/16 in virtual mode.
This field is immutable. | | | @@ -590,7 +590,7 @@ _Appears in:_ | `defaultPriorityClass` _string_ | DefaultPriorityClass specifies the priorityClassName applied to all pods of all clusters in the target Namespace. | | | | `defaultServerAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | DefaultServerAffinity specifies the affinity rules applied to server pods of all clusters in the target Namespace.
This includes both node affinity and pod affinity/anti-affinity rules. | | | | `defaultAgentAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | DefaultAgentAffinity specifies the affinity rules applied to agent pods of all clusters in the target Namespace.
This includes both node affinity and pod affinity/anti-affinity rules. | | | -| `allowedMode` _[ClusterMode](#clustermode)_ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". | shared | Enum: [shared virtual]
| +| `allowedMode` _[ClusterMode](#clustermode)_ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". | shared | Enum: [shared virtual hcp]
| | `disableNetworkPolicy` _boolean_ | DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. | | | | `podSecurityAdmissionLevel` _[PodSecurityAdmissionLevel](#podsecurityadmissionlevel)_ | PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. | | Enum: [privileged baseline restricted]
| | `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{ \} | | diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index a3758fb1..59f47198 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -624,14 +624,6 @@ type ClusterStatus struct { // +optional KubeletPort int `json:"kubeletPort,omitempty"` - // HCPRegistration is a copy-pasteable K3s installer command that external - // (BYO) nodes can run to register against an HCP-mode cluster. - // Only populated when Mode is "hcp" and an externally-routable endpoint - // (NodePort, LoadBalancer or Ingress) is configured. - // - // +optional - HCPRegistration string `json:"hcpRegistration,omitempty"` - // Conditions are the individual conditions for the cluster set. // // +optional diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go index dff7fab8..0c5ce04b 100644 --- a/pkg/controller/cluster/hcp.go +++ b/pkg/controller/cluster/hcp.go @@ -29,7 +29,7 @@ import ( func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error { log := ctrl.LoggerFrom(ctx) - url, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0) + _, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0) if err != nil { return err } @@ -44,33 +44,11 @@ func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster * Reason: ReasonHCPNoExternalEndpoint, Message: "HCP cluster has no external endpoint; set spec.expose.nodePort, spec.expose.loadBalancer or spec.expose.ingress so external nodes can reach the API server", }) - - cluster.Status.HCPRegistration = "" - - return nil } - version := cluster.Spec.Version - if version == "" { - version = cluster.Status.HostVersion - } - - cluster.Status.HCPRegistration = hcpRegistrationCommand(version, url, token) - return nil } -// hcpRegistrationCommand returns the standard K3s installer one-liner an -// end-user can copy onto an external host to join an HCP cluster. -func hcpRegistrationCommand(version, serverURL, token string) string { - if version == "" { - return fmt.Sprintf("curl -sfL https://get.k3s.io | K3S_URL=%s K3S_TOKEN=%s sh -", serverURL, token) - } - - return fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s K3S_URL=%s K3S_TOKEN=%s sh -", - version, serverURL, token) -} - // selectNonLoopbackSAN returns the first non-loopback address from the // cluster's TLS SANs, preferring spec.TLSSANs then falling back to status.TLSSANs. // Returns empty string if no non-loopback address is found. diff --git a/pkg/controller/cluster/hcp_test.go b/pkg/controller/cluster/hcp_test.go index 6587b0aa..da1b5f57 100644 --- a/pkg/controller/cluster/hcp_test.go +++ b/pkg/controller/cluster/hcp_test.go @@ -2,7 +2,6 @@ package cluster import ( "context" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -19,38 +18,6 @@ import ( "github.com/rancher/k3k/pkg/controller/cluster/server" ) -func Test_hcpRegistrationCommand(t *testing.T) { - tests := []struct { - name string - version string - serverURL string - token string - want string - }{ - { - name: "with version", - version: "v1.33.1-k3s1", - serverURL: "https://1.2.3.4:30443", - token: "abcd1234", - want: "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.33.1-k3s1 K3S_URL=https://1.2.3.4:30443 K3S_TOKEN=abcd1234 sh -", - }, - { - name: "without version", - version: "", - serverURL: "https://hcp.example.com", - token: "tok", - want: "curl -sfL https://get.k3s.io | K3S_URL=https://hcp.example.com K3S_TOKEN=tok sh -", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := hcpRegistrationCommand(tt.version, tt.serverURL, tt.token) - assert.Equal(t, tt.want, got) - }) - } -} - func Test_ensureHCPRegistration(t *testing.T) { scheme := runtime.NewScheme() require.NoError(t, corev1.AddToScheme(scheme)) @@ -68,33 +35,6 @@ func Test_ensureHCPRegistration(t *testing.T) { }, } - t.Run("nodeport service produces ready-to-copy command", func(t *testing.T) { - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: server.ServiceName(cluster.Name), - Namespace: cluster.Namespace, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeNodePort, - ClusterIP: "10.43.0.50", - Ports: []corev1.ServicePort{ - {Name: "k3s-server-port", Port: 443, NodePort: 31001}, - }, - }, - } - - fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build() - r := &ClusterReconciler{Client: fakeClient} - - c := cluster.DeepCopy() - require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "join-token-xyz")) - - assert.Contains(t, c.Status.HCPRegistration, "K3S_URL=https://hcp.example.com:31001") - assert.Contains(t, c.Status.HCPRegistration, "K3S_TOKEN=join-token-xyz") - assert.Contains(t, c.Status.HCPRegistration, "INSTALL_K3S_VERSION=v1.33.1-k3s1") - assert.True(t, strings.HasPrefix(c.Status.HCPRegistration, "curl -sfL https://get.k3s.io")) - }) - t.Run("clusterip-only service sets degraded condition and clears registration", func(t *testing.T) { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -114,11 +54,8 @@ func Test_ensureHCPRegistration(t *testing.T) { r := &ClusterReconciler{Client: fakeClient} c := cluster.DeepCopy() - c.Status.HCPRegistration = "stale-value" require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "ignored")) - assert.Empty(t, c.Status.HCPRegistration) - cond := meta.FindStatusCondition(c.Status.Conditions, ConditionReady) require.NotNil(t, cond) assert.Equal(t, metav1.ConditionFalse, cond.Status) From 9c06d4483512f2dec1f4b25f0d93d782cf9e0731 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 10 Jun 2026 14:53:40 +0200 Subject: [PATCH 08/36] fix tests --- pkg/controller/cluster/server/config_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/controller/cluster/server/config_test.go b/pkg/controller/cluster/server/config_test.go index fe873d50..ede7a79b 100644 --- a/pkg/controller/cluster/server/config_test.go +++ b/pkg/controller/cluster/server/config_test.go @@ -2,9 +2,10 @@ package server import ( "fmt" - "reflect" "testing" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" @@ -43,6 +44,9 @@ func Test_BuildServerConfig(t *testing.T) { Name: testClusterName, Namespace: testClusterNamespace, }, + Spec: v1beta1.ClusterSpec{ + Mode: v1beta1.SharedClusterMode, + }, Status: v1beta1.ClusterStatus{ ClusterCIDR: defaultSharedClusterCIDR, ServiceCIDR: defaultSharedServiceCIDR, @@ -71,6 +75,9 @@ func Test_BuildServerConfig(t *testing.T) { Name: testClusterName, Namespace: testClusterNamespace, }, + Spec: v1beta1.ClusterSpec{ + Mode: v1beta1.SharedClusterMode, + }, Status: v1beta1.ClusterStatus{ ClusterCIDR: defaultSharedClusterCIDR, ServiceCIDR: defaultSharedServiceCIDR, @@ -158,6 +165,7 @@ func Test_BuildServerConfig(t *testing.T) { Namespace: testClusterNamespace, }, Spec: v1beta1.ClusterSpec{ + Mode: v1beta1.SharedClusterMode, ClusterDNS: testClusterDNS, }, Status: v1beta1.ClusterStatus{ @@ -186,9 +194,7 @@ func Test_BuildServerConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { serverConfig := buildServerConfig(tt.args.cluster, tt.args.initServer, tt.args.serviceIP, tt.args.token) - if !reflect.DeepEqual(tt.expectedData, serverConfig) { - t.Errorf("found %v, expected %v", serverConfig, tt.expectedData) - } + assert.Equal(t, tt.expectedData, serverConfig) }) } } From 51a7ca0107ea5f961991e36158aa21dce40f4dd2 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 10 Jun 2026 16:14:31 +0200 Subject: [PATCH 09/36] added HCP conformance tests --- .github/workflows/test-conformance-hcp.yaml | 342 ++++++++++++++++++++ pkg/controller/cluster/cluster.go | 6 +- pkg/controller/cluster/hcp.go | 75 ++++- pkg/controller/cluster/hcp_test.go | 2 +- 4 files changed, 421 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test-conformance-hcp.yaml diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml new file mode 100644 index 00000000..b2c8caf5 --- /dev/null +++ b/.github/workflows/test-conformance-hcp.yaml @@ -0,0 +1,342 @@ +name: Conformance Tests - HCP Mode + +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + inputs: + k3k_version: + description: 'K3k version to test (e.g. v1.1.0). Leave empty to build from source.' + required: false + type: string + k8s_version: + description: 'Kubernetes version to test' + required: false + type: choice + options: + - "" + - "v1.34.6" + - "v1.35.3" + +permissions: + contents: read + +env: + K8S_VERSIONS: "v1.34.6,v1.35.3" + HELM_VERSION: v4.1.3 + HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700 + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + k8s_versions: ${{ steps.set-matrix.outputs.k8s_versions }} + steps: + - id: set-matrix + run: | + if [[ -z "${{ inputs.k8s_version }}" ]]; then + JSON_ARRAY=$(jq -nc '"${{ env.K8S_VERSIONS }}" | split(",")') + echo "k8s_versions=${JSON_ARRAY}" >> "$GITHUB_OUTPUT" + else + echo "k8s_versions=[\"${{ inputs.k8s_version }}\"]" >> "$GITHUB_OUTPUT" + fi + + conformance: + needs: setup + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + k8s_version: ${{ fromJSON(needs.setup.outputs.k8s_versions) }} + + env: + KUBERNETES_VERSION: ${{ matrix.k8s_version }} + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version-file: go.mod + + - name: Install helm + env: + FILENAME: helm.tar.gz + run: | + curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz + echo "${{ env.HELM_CHECKSUM_AMD64 }} ${{ env.FILENAME }}" | sha256sum --check + tar -xvzf ${{ env.FILENAME }} linux-amd64/helm + sudo install -m 755 linux-amd64/helm /usr/local/bin/helm + + rm -fr "${{ env.FILENAME }}" linux-amd64/helm + + - name: Install hydrophone + run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0 + + - name: Install k3s + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 + run: | + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + + kubectl cluster-info + kubectl get nodes + + echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV + + - name: Setup K3k (from source) + if: inputs.k3k_version == '' + run: | + export REPO=ttl.sh/$(uuidgen) + export VERSION=1h + + make build + make package + make push + make install + + # add k3kcli to $PATH + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + + - name: Setup K3k (from release) + if: inputs.k3k_version != '' + run: | + K3K_VERSION="${{ inputs.k3k_version }}" + CHART_VERSION="${K3K_VERSION#v}" + + helm repo add k3k https://rancher.github.io/k3k + helm repo update + helm install --namespace k3k-system --create-namespace --version "${CHART_VERSION}" k3k k3k/k3k + + wget -qO k3kcli "https://github.com/rancher/k3k/releases/download/${{ inputs.k3k_version }}/k3kcli-linux-amd64" + sudo mv k3kcli /usr/local/bin/k3kcli + sudo chmod +x /usr/local/bin/k3kcli + + - name: Wait for K3k controller + run: | + echo "Wait for K3k controller deployment to be available" + kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m + + - name: Check k3kcli + run: k3kcli -v + + - name: Create virtual cluster + run: | + kubectl create namespace k3k-mycluster + + cat < user-data + #cloud-config + users: + - name: ubuntu + ssh_authorized_keys: + - $(cat ./id_rsa.pub) + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + shell: /bin/bash + EOF + + # Create separate seed images for each worker to avoid locking issues + cloud-localds seed-1.img user-data + cloud-localds seed-2.img user-data + + - name: Create Worker Disks + run: | + qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G + qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-2.qcow2 20G + + - name: Launch Worker VMs + run: | + # Launch Worker 1 + sudo qemu-system-x86_64 \ + -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ + -drive file=worker-1.qcow2,if=virtio \ + -drive file=seed-1.img,format=raw,if=virtio \ + -net nic,model=virtio,macaddr=52:54:00:12:34:56 \ + -net user,hostfwd=tcp::2222-:22 \ + & + + # Wait a moment before launching the second VM + sleep 5 + + # Launch Worker 2 + sudo qemu-system-x86_64 \ + -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ + -drive file=worker-2.qcow2,if=virtio \ + -drive file=seed-2.img,format=raw,if=virtio \ + -net nic,model=virtio,macaddr=52:54:00:12:34:57 \ + -net user,hostfwd=tcp::2223-:22 \ + & + + - name: Wait for SSH Availability + run: | + echo "Waiting for Worker 1 (Port 2222) to respond..." + timeout 120s bash -c ' + until ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done + ' + + echo "Waiting for Worker 2 (Port 2223) to respond..." + timeout 120s bash -c ' + until ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done + ' + + echo "Both VMs are up and running!" + + echo "Testing connectivity from VMs to K3k API server..." + ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'Worker 1 connectivity test failed'" + + ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'Worker 2 connectivity test failed'" + + ###################### + + - name: Join Workers to K3k Control Plane + run: | + K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) + + echo "Registering Worker 1..." + ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh - &" & + WORKER1_PID=$! + + echo "Registering Worker 2..." + ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh - &" & + WORKER2_PID=$! + + echo "Waiting for both workers to complete registration..." + wait $WORKER1_PID + wait $WORKER2_PID + echo "Both workers registration initiated" + + - name: Verify Cluster Nodes + env: + KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml + run: | + echo "Monitoring K3k Virtual Cluster for Node registration..." + + kubectl get pod -A + kubectl get nodes + + timeout 180s bash -c ' + until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 2 ]; do + echo "Waiting for both worker nodes to show Ready..." + kubectl get nodes || true + sleep 5 + done + ' + + ###################### + + - name: Run conformance tests + run: | + hydrophone --conformance --parallel 4 \ + --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ + --output-dir /tmp + + - name: Collect logs + if: always() + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + run: | + journalctl -u k3s -o cat --no-pager > /tmp/k3s.log + kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log + + - name: Archive K3s logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3s-${{ matrix.k8s_version }}-logs + path: /tmp/k3s.log + + - name: Archive K3k logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3k-${{ matrix.k8s_version }}-logs + path: /tmp/k3k.log + + - name: Archive conformance logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: conformance-${{ matrix.k8s_version }}-logs + path: /tmp/e2e.log + + - name: Archive results + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: conformance-${{ matrix.k8s_version }}-results + path: /tmp/junit_01.xml + + - name: Job Summary + if: always() + run: | + echo '## 📊 Conformance Tests Results (${{ matrix.k8s_version }})' >> $GITHUB_STEP_SUMMARY + echo '| Passed | Failed | Pending | Skipped |' >> $GITHUB_STEP_SUMMARY + echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY + + RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3) + RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g') + echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY + + # only include failed tests section if there are any + if grep -q '\[FAIL\]' /tmp/e2e.log; then + echo '' >> $GITHUB_STEP_SUMMARY + echo '### Failed Tests' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep '\[FAIL\]' /tmp/e2e.log >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index 91cd7ce5..9a4601fd 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -454,13 +454,17 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus // reconciler is disabled for HCP) so external-node pods can reach the // in-cluster apiserver ClusterIP. if cluster.Spec.Mode == v1beta1.HCPClusterMode { - if err := c.ensureHCPRegistration(ctx, cluster, token); err != nil { + if err := c.ensureHCPRegistration(ctx, cluster); err != nil { return err } if err := c.ensureHCPKubernetesEndpointSlice(ctx, cluster); err != nil { return err } + + if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil { + return err + } } // Important: if you need to call the Server API of the Virtual Cluster diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go index 0c5ce04b..127c5e0c 100644 --- a/pkg/controller/cluster/hcp.go +++ b/pkg/controller/cluster/hcp.go @@ -26,7 +26,7 @@ import ( // LoadBalancer or Ingress configured) the command cannot be built; in that // case the Ready condition is set to False with reason HCPNoExternalEndpoint // so the operator surfaces the problem without failing the reconciliation. -func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error { +func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster) error { log := ctrl.LoggerFrom(ctx) _, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0) @@ -143,11 +143,11 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context } _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpointSlice, func() error { - // Ensure the service-name label is set if endpointSlice.Labels == nil { endpointSlice.Labels = make(map[string]string) } + // Ensure the service-name label is set endpointSlice.Labels[discoveryv1.LabelServiceName] = "kubernetes" endpointSlice.AddressType = addressType @@ -182,6 +182,77 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context return nil } +func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error { + log := ctrl.LoggerFrom(ctx) + + rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0) + if err != nil { + return err + } + + if !external { + // ensureHCPRegistration already surfaces this via Ready=False; + // nothing for us to do here. + return nil + } + + host, port, err := parseHCPHostPort(rawURL) + if err != nil { + return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err) + } + + addr, err := hcpEndpointAddress(host) + if err != nil { + return err + } + + virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace) + if err != nil { + return fmt.Errorf("creating virtual cluster client: %w", err) + } + + //nolint:staticcheck // SA1019 corev1.Endpoints is deprecated in v1.33+, but needed in the Conformance tests + // We are already using the discoveryv1.EndpointSlice + endpoints := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubernetes", + Namespace: metav1.NamespaceDefault, + }, + } + + _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error { + if endpoints.Labels == nil { + endpoints.Labels = make(map[string]string) + } + + // Ensure the skip-mirror label is set + endpoints.Labels[discoveryv1.LabelSkipMirror] = "true" + + //nolint:staticcheck // SA1019 corev1.EndpointSubset is deprecated in v1.33+, but needed in the Conformance tests + endpoints.Subsets = []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{addr}, + Ports: []corev1.EndpointPort{ + { + Name: "https", + Port: port, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + + return nil + }) + if err != nil { + return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err) + } + + log.V(1).Info("HCP kubernetes endpoints reconciled", "address", addr.IP, "hostname", addr.Hostname, "port", port) + + return nil +} + // parseHCPHostPort extracts the host and port from a server URL produced by // server.ServerURL. The port defaults to 443 when omitted. func parseHCPHostPort(rawURL string) (string, int32, error) { diff --git a/pkg/controller/cluster/hcp_test.go b/pkg/controller/cluster/hcp_test.go index da1b5f57..c85ef986 100644 --- a/pkg/controller/cluster/hcp_test.go +++ b/pkg/controller/cluster/hcp_test.go @@ -54,7 +54,7 @@ func Test_ensureHCPRegistration(t *testing.T) { r := &ClusterReconciler{Client: fakeClient} c := cluster.DeepCopy() - require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "ignored")) + require.NoError(t, r.ensureHCPRegistration(context.Background(), c)) cond := meta.FindStatusCondition(c.Status.Conditions, ConditionReady) require.NotNil(t, cond) From 0c2beda44644848105c152b03c2e9d87b05c910e Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 10 Jun 2026 21:23:13 +0200 Subject: [PATCH 10/36] fix node-name and added log --- .github/workflows/test-conformance-hcp.yaml | 22 ++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index b2c8caf5..f8651888 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -239,6 +239,17 @@ jobs: ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'Worker 2 connectivity test failed'" + - name: Verify Worker VM Configuration + run: | + echo "=== Worker 1 Configuration ===" + ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "echo 'Hostname:' \$(hostname) && echo 'IP Address:' \$(ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && echo 'Gateway:' \$(ip route | grep default)" + + echo "" + echo "=== Worker 2 Configuration ===" + ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "echo 'Hostname:' \$(hostname) && echo 'IP Address:' \$(ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && echo 'Gateway:' \$(ip route | grep default)" + ###################### - name: Join Workers to K3k Control Plane @@ -247,18 +258,11 @@ jobs: echo "Registering Worker 1..." ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh - &" & - WORKER1_PID=$! + "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} INSTALL_K3S_EXEC='--node-name=worker-1' sh -" echo "Registering Worker 2..." ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh - &" & - WORKER2_PID=$! - - echo "Waiting for both workers to complete registration..." - wait $WORKER1_PID - wait $WORKER2_PID - echo "Both workers registration initiated" + "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} INSTALL_K3S_EXEC='--node-name=worker-2' sh -" - name: Verify Cluster Nodes env: From 104bb5bccd0d826ff4ebb0735e44153cb0ae464e Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Thu, 11 Jun 2026 01:48:32 +0200 Subject: [PATCH 11/36] single worker --- .github/workflows/test-conformance-hcp.yaml | 262 +++++++++++++++++++- 1 file changed, 260 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index f8651888..450d0d63 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -41,6 +41,264 @@ jobs: echo "k8s_versions=[\"${{ inputs.k8s_version }}\"]" >> "$GITHUB_OUTPUT" fi + conformance-single: + needs: setup + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + k8s_version: ${{ fromJSON(needs.setup.outputs.k8s_versions) }} + + env: + KUBERNETES_VERSION: ${{ matrix.k8s_version }} + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version-file: go.mod + + - name: Install helm + env: + FILENAME: helm.tar.gz + run: | + curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz + echo "${{ env.HELM_CHECKSUM_AMD64 }} ${{ env.FILENAME }}" | sha256sum --check + tar -xvzf ${{ env.FILENAME }} linux-amd64/helm + sudo install -m 755 linux-amd64/helm /usr/local/bin/helm + + rm -fr "${{ env.FILENAME }}" linux-amd64/helm + + - name: Install hydrophone + run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0 + + - name: Install k3s + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 + run: | + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + + kubectl cluster-info + kubectl get nodes + + echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV + + - name: Setup K3k (from source) + if: inputs.k3k_version == '' + run: | + export REPO=ttl.sh/$(uuidgen) + export VERSION=1h + + make build + make package + make push + make install + + # add k3kcli to $PATH + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + + - name: Setup K3k (from release) + if: inputs.k3k_version != '' + run: | + K3K_VERSION="${{ inputs.k3k_version }}" + CHART_VERSION="${K3K_VERSION#v}" + + helm repo add k3k https://rancher.github.io/k3k + helm repo update + helm install --namespace k3k-system --create-namespace --version "${CHART_VERSION}" k3k k3k/k3k + + wget -qO k3kcli "https://github.com/rancher/k3k/releases/download/${{ inputs.k3k_version }}/k3kcli-linux-amd64" + sudo mv k3kcli /usr/local/bin/k3kcli + sudo chmod +x /usr/local/bin/k3kcli + + - name: Wait for K3k controller + run: | + echo "Wait for K3k controller deployment to be available" + kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m + + - name: Check k3kcli + run: k3kcli -v + + - name: Create virtual cluster + run: | + kubectl create namespace k3k-mycluster + + cat < user-data + #cloud-config + users: + - name: ubuntu + ssh_authorized_keys: + - $(cat ./id_rsa.pub) + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + shell: /bin/bash + EOF + + cloud-localds seed.img user-data + + - name: Create Worker Disk + run: | + qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G + + - name: Launch Worker VM + run: | + sudo qemu-system-x86_64 \ + -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ + -drive file=worker-1.qcow2,if=virtio \ + -drive file=seed.img,format=raw,if=virtio \ + -net nic,model=virtio,macaddr=52:54:00:12:34:56 \ + -net user,hostfwd=tcp::2222-:22 \ + & + + - name: Wait for SSH Availability + run: | + echo "Waiting for Worker (Port 2222) to respond..." + timeout 120s bash -c ' + until ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done + ' + + echo "Worker VM is up and running!" + + echo "Testing connectivity from VM to K3k API server..." + ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'VM connectivity test failed'" + + - name: Join Worker to K3k Control Plane + run: | + K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) + + echo "Registering Worker..." + ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh -" + + - name: Verify Cluster Nodes + env: + KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml + run: | + echo "Monitoring K3k Virtual Cluster for Node registration..." + + kubectl get pod -A + kubectl get nodes + + timeout 180s bash -c ' + until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 1 ]; do + echo "Waiting for worker node to show Ready..." + kubectl get nodes || true + sleep 5 + done + ' + + - name: Run conformance tests + run: | + hydrophone --conformance --parallel 4 \ + --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ + --output-dir /tmp + + - name: Collect logs + if: always() + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + run: | + journalctl -u k3s -o cat --no-pager > /tmp/k3s.log + kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log + + - name: Archive K3s logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3s-single-${{ matrix.k8s_version }}-logs + path: /tmp/k3s.log + + - name: Archive K3k logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3k-single-${{ matrix.k8s_version }}-logs + path: /tmp/k3k.log + + - name: Archive conformance logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: conformance-single-${{ matrix.k8s_version }}-logs + path: /tmp/e2e.log + + - name: Archive results + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: conformance-single-${{ matrix.k8s_version }}-results + path: /tmp/junit_01.xml + + - name: Job Summary + if: always() + run: | + echo '## 📊 Conformance Tests Results - Single Worker (${{ matrix.k8s_version }})' >> $GITHUB_STEP_SUMMARY + echo '| Passed | Failed | Pending | Skipped |' >> $GITHUB_STEP_SUMMARY + echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY + + RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3) + RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g') + echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY + + # only include failed tests section if there are any + if grep -q '\[FAIL\]' /tmp/e2e.log; then + echo '' >> $GITHUB_STEP_SUMMARY + echo '### Failed Tests' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep '\[FAIL\]' /tmp/e2e.log >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + conformance: needs: setup runs-on: ubuntu-latest @@ -243,12 +501,12 @@ jobs: run: | echo "=== Worker 1 Configuration ===" ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "echo 'Hostname:' \$(hostname) && echo 'IP Address:' \$(ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && echo 'Gateway:' \$(ip route | grep default)" + "echo 'Hostname:' \$(hostname) && echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && echo 'Gateway:' \$(ip route | grep default)" echo "" echo "=== Worker 2 Configuration ===" ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "echo 'Hostname:' \$(hostname) && echo 'IP Address:' \$(ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && echo 'Gateway:' \$(ip route | grep default)" + "echo 'Hostname:' \$(hostname) && echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && echo 'Gateway:' \$(ip route | grep default)" ###################### From 0a081c6504563d3be78240ba887dc3e6f71568f8 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Thu, 11 Jun 2026 15:22:09 +0200 Subject: [PATCH 12/36] reduce parallelism --- .github/workflows/test-conformance-hcp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index 450d0d63..21eb8d11 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -543,7 +543,7 @@ jobs: - name: Run conformance tests run: | - hydrophone --conformance --parallel 4 \ + hydrophone --conformance --parallel 2 \ --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ --output-dir /tmp From ee61eae71f698434094a6284b02e98fa96783270 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Fri, 12 Jun 2026 18:27:59 +0200 Subject: [PATCH 13/36] warning for hcp --- cli/cmds/cluster_create.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cli/cmds/cluster_create.go b/cli/cmds/cluster_create.go index 90ae6498..cbfcd654 100644 --- a/cli/cmds/cluster_create.go +++ b/cli/cmds/cluster_create.go @@ -84,10 +84,14 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm return errors.New("invalid cluster name") } - if (config.mode == string(v1beta1.SharedClusterMode) || config.mode == string(v1beta1.HCPClusterMode)) && config.agents != 0 { + if config.agents > 0 && config.mode != string(v1beta1.VirtualClusterMode) { return errors.New("invalid flag, --agents flag is only allowed in virtual mode") } + if config.mode == string(v1beta1.HCPClusterMode) { + logrus.Warn("Mode 'hcp' is experimental") + } + namespace := appCtx.Namespace(name) if err := createNamespace(ctx, client, namespace, config.policy); err != nil { From d422898d55411e975335c5a267c097cdc3e2e0ab Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Fri, 12 Jun 2026 19:06:18 +0200 Subject: [PATCH 14/36] fix multi-VM HCP conformance test networking Both QEMU workers booted with `-net user` and ended up registering the same InternalIP (10.0.2.15) because each VM gets its own isolated NAT slirp. Flannel propagated this to `public-ip` on both nodes, so VXLAN could not tunnel between workers and any cross-node pod traffic broke (89 failed / 335 passed of 424 conformance specs). Replace user-mode networking with a Linux bridge (k3kbr0, 192.168.100.0/24) and one TAP device per VM, so the two workers share an L2 segment with unique routable IPs. NAT outbound from the bridge keeps internet access working for image pulls. Also set unique hostnames via cloud-init (worker-1/worker-2) and drop the `--node-name` flag from INSTALL_K3S_EXEC, since k3s now picks the correct node name from the OS hostname on its own. Bump hydrophone back to `--parallel 4` to match the single-VM job (parallelism was reduced earlier when the failure was thought to be resource-related). --- .github/workflows/test-conformance-hcp.yaml | 132 ++++++++++++++------ 1 file changed, 91 insertions(+), 41 deletions(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index 21eb8d11..84bada57 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -398,7 +398,7 @@ jobs: mode: hcp tlsSANs: - "127.0.0.1" - - "10.0.2.2" + - "192.168.100.1" expose: nodePort: serverPort: 30001 @@ -409,8 +409,8 @@ jobs: k3kcli kubeconfig generate --name mycluster - export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml - + export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml + kubectl cluster-info kubectl get nodes kubectl get pods -A @@ -425,28 +425,78 @@ jobs: kvm-ok + - name: Set up bridge network for VMs + run: | + # Create a bridge so both VMs share an L2 segment with unique routable IPs. + # Required because QEMU `-net user` gives every VM the same 10.0.2.15 NAT + # address, which breaks flannel VXLAN between workers. + sudo ip link add name k3kbr0 type bridge + sudo ip addr add 192.168.100.1/24 dev k3kbr0 + sudo ip link set k3kbr0 up + + # NAT outbound so VMs can reach the internet (image pulls etc). + sudo sysctl -w net.ipv4.ip_forward=1 + sudo iptables -t nat -A POSTROUTING -s 192.168.100.0/24 ! -o k3kbr0 -j MASQUERADE + sudo iptables -A FORWARD -i k3kbr0 -j ACCEPT + sudo iptables -A FORWARD -o k3kbr0 -j ACCEPT + + # One TAP per VM, attached to the bridge. + sudo ip tuntap add tap-w1 mode tap + sudo ip link set tap-w1 master k3kbr0 + sudo ip link set tap-w1 up + + sudo ip tuntap add tap-w2 mode tap + sudo ip link set tap-w2 master k3kbr0 + sudo ip link set tap-w2 up + - name: Download Base Cloud Image run: | wget -q https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img - - - name: Generate SSH Key and Cloud-Init Seed + + - name: Generate SSH Key and Cloud-Init Seeds run: | ssh-keygen -t rsa -b 4096 -f ./id_rsa -N "" + PUBKEY="$(cat ./id_rsa.pub)" - # Define the cloud-init config to authorize our new key - cat < user-data + # Per-VM cloud-init: each worker gets a unique hostname and a static IP + # on the bridge subnet via netplan. + for i in 1 2; do + IP="192.168.100.1${i}" # 192.168.100.11 / 192.168.100.12 + cat < user-data-${i} #cloud-config + hostname: worker-${i} + preserve_hostname: false + manage_etc_hosts: true + # Stop cloud-init from generating its own DHCP netplan that would + # conflict with the static one we write in write_files below. + network: + config: disabled users: - name: ubuntu ssh_authorized_keys: - - $(cat ./id_rsa.pub) + - ${PUBKEY} sudo: ['ALL=(ALL) NOPASSWD:ALL'] shell: /bin/bash + write_files: + - path: /etc/netplan/50-static.yaml + permissions: '0600' + content: | + network: + version: 2 + ethernets: + ens3: + dhcp4: false + addresses: [${IP}/24] + routes: + - to: default + via: 192.168.100.1 + nameservers: + addresses: [8.8.8.8, 1.1.1.1] + runcmd: + - netplan apply EOF - - # Create separate seed images for each worker to avoid locking issues - cloud-localds seed-1.img user-data - cloud-localds seed-2.img user-data + cloud-localds seed-${i}.img user-data-${i} + done - name: Create Worker Disks run: | @@ -455,58 +505,58 @@ jobs: - name: Launch Worker VMs run: | - # Launch Worker 1 + # Launch Worker 1 — attached to tap-w1 on k3kbr0. sudo qemu-system-x86_64 \ -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ -drive file=worker-1.qcow2,if=virtio \ -drive file=seed-1.img,format=raw,if=virtio \ - -net nic,model=virtio,macaddr=52:54:00:12:34:56 \ - -net user,hostfwd=tcp::2222-:22 \ + -netdev tap,id=net0,ifname=tap-w1,script=no,downscript=no \ + -device virtio-net-pci,netdev=net0,mac=52:54:00:12:34:56 \ & # Wait a moment before launching the second VM sleep 5 - # Launch Worker 2 + # Launch Worker 2 — attached to tap-w2 on k3kbr0. sudo qemu-system-x86_64 \ -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ -drive file=worker-2.qcow2,if=virtio \ -drive file=seed-2.img,format=raw,if=virtio \ - -net nic,model=virtio,macaddr=52:54:00:12:34:57 \ - -net user,hostfwd=tcp::2223-:22 \ + -netdev tap,id=net0,ifname=tap-w2,script=no,downscript=no \ + -device virtio-net-pci,netdev=net0,mac=52:54:00:12:34:57 \ & - name: Wait for SSH Availability run: | - echo "Waiting for Worker 1 (Port 2222) to respond..." - timeout 120s bash -c ' - until ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done + echo "Waiting for Worker 1 (192.168.100.11) to respond..." + timeout 180s bash -c ' + until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@192.168.100.11 true 2>/dev/null; do sleep 3; done ' - echo "Waiting for Worker 2 (Port 2223) to respond..." - timeout 120s bash -c ' - until ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done + echo "Waiting for Worker 2 (192.168.100.12) to respond..." + timeout 180s bash -c ' + until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@192.168.100.12 true 2>/dev/null; do sleep 3; done ' echo "Both VMs are up and running!" echo "Testing connectivity from VMs to K3k API server..." - ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'Worker 1 connectivity test failed'" + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \ + "curl -kv --max-time 10 https://192.168.100.1:30001/readyz || echo 'Worker 1 connectivity test failed'" - ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'Worker 2 connectivity test failed'" + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \ + "curl -kv --max-time 10 https://192.168.100.1:30001/readyz || echo 'Worker 2 connectivity test failed'" - name: Verify Worker VM Configuration run: | - echo "=== Worker 1 Configuration ===" - ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "echo 'Hostname:' \$(hostname) && echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && echo 'Gateway:' \$(ip route | grep default)" - - echo "" - echo "=== Worker 2 Configuration ===" - ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "echo 'Hostname:' \$(hostname) && echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && echo 'Gateway:' \$(ip route | grep default)" + for IP in 192.168.100.11 192.168.100.12; do + echo "=== Worker at ${IP} ===" + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ + "echo 'Hostname:' \$(hostname) && \ + echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && \ + echo 'Gateway:' \$(ip route | grep default)" + echo "" + done ###################### @@ -515,12 +565,12 @@ jobs: K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) echo "Registering Worker 1..." - ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} INSTALL_K3S_EXEC='--node-name=worker-1' sh -" + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \ + "curl -sfL https://get.k3s.io | K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -" echo "Registering Worker 2..." - ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} INSTALL_K3S_EXEC='--node-name=worker-2' sh -" + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \ + "curl -sfL https://get.k3s.io | K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -" - name: Verify Cluster Nodes env: @@ -543,7 +593,7 @@ jobs: - name: Run conformance tests run: | - hydrophone --conformance --parallel 2 \ + hydrophone --conformance --parallel 4 \ --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ --output-dir /tmp From e4da450e4b3ef7982732471182fe9cc017207272 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Fri, 12 Jun 2026 20:39:37 +0200 Subject: [PATCH 15/36] removed single worker --- .github/workflows/test-conformance-hcp.yaml | 262 -------------------- 1 file changed, 262 deletions(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index 84bada57..38378b0d 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -41,264 +41,6 @@ jobs: echo "k8s_versions=[\"${{ inputs.k8s_version }}\"]" >> "$GITHUB_OUTPUT" fi - conformance-single: - needs: setup - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - k8s_version: ${{ fromJSON(needs.setup.outputs.k8s_versions) }} - - env: - KUBERNETES_VERSION: ${{ matrix.k8s_version }} - - steps: - - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - fetch-depth: 0 - fetch-tags: true - - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 - with: - go-version-file: go.mod - - - name: Install helm - env: - FILENAME: helm.tar.gz - run: | - curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz - echo "${{ env.HELM_CHECKSUM_AMD64 }} ${{ env.FILENAME }}" | sha256sum --check - tar -xvzf ${{ env.FILENAME }} linux-amd64/helm - sudo install -m 755 linux-amd64/helm /usr/local/bin/helm - - rm -fr "${{ env.FILENAME }}" linux-amd64/helm - - - name: Install hydrophone - run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0 - - - name: Install k3s - env: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml - K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 - run: | - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - - - kubectl cluster-info - kubectl get nodes - - echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV - - - name: Setup K3k (from source) - if: inputs.k3k_version == '' - run: | - export REPO=ttl.sh/$(uuidgen) - export VERSION=1h - - make build - make package - make push - make install - - # add k3kcli to $PATH - echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - - - name: Setup K3k (from release) - if: inputs.k3k_version != '' - run: | - K3K_VERSION="${{ inputs.k3k_version }}" - CHART_VERSION="${K3K_VERSION#v}" - - helm repo add k3k https://rancher.github.io/k3k - helm repo update - helm install --namespace k3k-system --create-namespace --version "${CHART_VERSION}" k3k k3k/k3k - - wget -qO k3kcli "https://github.com/rancher/k3k/releases/download/${{ inputs.k3k_version }}/k3kcli-linux-amd64" - sudo mv k3kcli /usr/local/bin/k3kcli - sudo chmod +x /usr/local/bin/k3kcli - - - name: Wait for K3k controller - run: | - echo "Wait for K3k controller deployment to be available" - kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m - - - name: Check k3kcli - run: k3kcli -v - - - name: Create virtual cluster - run: | - kubectl create namespace k3k-mycluster - - cat < user-data - #cloud-config - users: - - name: ubuntu - ssh_authorized_keys: - - $(cat ./id_rsa.pub) - sudo: ['ALL=(ALL) NOPASSWD:ALL'] - shell: /bin/bash - EOF - - cloud-localds seed.img user-data - - - name: Create Worker Disk - run: | - qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G - - - name: Launch Worker VM - run: | - sudo qemu-system-x86_64 \ - -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ - -drive file=worker-1.qcow2,if=virtio \ - -drive file=seed.img,format=raw,if=virtio \ - -net nic,model=virtio,macaddr=52:54:00:12:34:56 \ - -net user,hostfwd=tcp::2222-:22 \ - & - - - name: Wait for SSH Availability - run: | - echo "Waiting for Worker (Port 2222) to respond..." - timeout 120s bash -c ' - until ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done - ' - - echo "Worker VM is up and running!" - - echo "Testing connectivity from VM to K3k API server..." - ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'VM connectivity test failed'" - - - name: Join Worker to K3k Control Plane - run: | - K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) - - echo "Registering Worker..." - ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh -" - - - name: Verify Cluster Nodes - env: - KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml - run: | - echo "Monitoring K3k Virtual Cluster for Node registration..." - - kubectl get pod -A - kubectl get nodes - - timeout 180s bash -c ' - until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 1 ]; do - echo "Waiting for worker node to show Ready..." - kubectl get nodes || true - sleep 5 - done - ' - - - name: Run conformance tests - run: | - hydrophone --conformance --parallel 4 \ - --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ - --output-dir /tmp - - - name: Collect logs - if: always() - env: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml - run: | - journalctl -u k3s -o cat --no-pager > /tmp/k3s.log - kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log - - - name: Archive K3s logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: k3s-single-${{ matrix.k8s_version }}-logs - path: /tmp/k3s.log - - - name: Archive K3k logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: k3k-single-${{ matrix.k8s_version }}-logs - path: /tmp/k3k.log - - - name: Archive conformance logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: conformance-single-${{ matrix.k8s_version }}-logs - path: /tmp/e2e.log - - - name: Archive results - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: conformance-single-${{ matrix.k8s_version }}-results - path: /tmp/junit_01.xml - - - name: Job Summary - if: always() - run: | - echo '## 📊 Conformance Tests Results - Single Worker (${{ matrix.k8s_version }})' >> $GITHUB_STEP_SUMMARY - echo '| Passed | Failed | Pending | Skipped |' >> $GITHUB_STEP_SUMMARY - echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY - - RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3) - RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g') - echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY - - # only include failed tests section if there are any - if grep -q '\[FAIL\]' /tmp/e2e.log; then - echo '' >> $GITHUB_STEP_SUMMARY - echo '### Failed Tests' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - grep '\[FAIL\]' /tmp/e2e.log >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - fi - conformance: needs: setup runs-on: ubuntu-latest @@ -415,8 +157,6 @@ jobs: kubectl get nodes kubectl get pods -A -################### - - name: Install Virtualization Dependencies run: | sudo apt-get update @@ -558,8 +298,6 @@ jobs: echo "" done - ###################### - - name: Join Workers to K3k Control Plane run: | K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) From 81bd5100b9718cb475a61aa8e4e3237945756e4f Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Sat, 13 Jun 2026 00:18:52 +0200 Subject: [PATCH 16/36] isolate test --- .github/workflows/test-conformance-hcp.yaml | 10 +++++++--- pkg/controller/cluster/server/config.go | 10 ---------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index 38378b0d..1c6dd6c2 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -327,14 +327,18 @@ jobs: done ' - ###################### - - name: Run conformance tests run: | - hydrophone --conformance --parallel 4 \ + hydrophone --focus 'should be able to change the type from (ClusterIP|NodePort) to ExternalName' \ --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ --output-dir /tmp + # - name: Run conformance tests + # run: | + # hydrophone --conformance --parallel 4 \ + # --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ + # --output-dir /tmp + - name: Collect logs if: always() env: diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index 428b808c..31a30549 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -93,16 +93,6 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok // no extra config for virtual mode } - // In hcp mode the apiserver pod IP is unreachable from external worker - // nodes, so the kube-apiserver's default lease-based endpoint reconciler - // would publish a broken default/kubernetes Endpoints (advertise-address + - // secure-port). Disable it so K3k can own that Endpoints object and point - // it at the externally-reachable host:port (NodePort / LB / Ingress). - if cluster.Spec.Mode == v1beta1.HCPClusterMode { - serverConfig.KubeApiServerArg = append(serverConfig.KubeApiServerArg, "endpoint-reconciler-type=none") - serverConfig.EgressSelectorMode = "cluster" - } - // In shared mode workloads run on the host cluster, so the apiserver pod // can reach them directly via the host pod network and the egress // selector is unnecessary. From dad40e9c405191d991e719731dc58dd8a5c6cb2b Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Sat, 13 Jun 2026 00:50:20 +0200 Subject: [PATCH 17/36] mtu and kubernetes worker version --- .github/workflows/test-conformance-hcp.yaml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index 1c6dd6c2..7d6951e2 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -170,8 +170,16 @@ jobs: # Create a bridge so both VMs share an L2 segment with unique routable IPs. # Required because QEMU `-net user` gives every VM the same 10.0.2.15 NAT # address, which breaks flannel VXLAN between workers. + # + # MTU is set to 1400 end-to-end (bridge, TAPs, VM ens3 via cloud-init). + # The path traverses flannel VXLAN inside the virtual cluster AND the + # host's k3s pod network — each strips ~50 bytes for VXLAN headers. + # Defaulting to 1500 across the stack leaves no headroom and PMTUD + # through nested NAT/forwarding is unreliable; 1400 fits both layers + # without fragmentation. sudo ip link add name k3kbr0 type bridge sudo ip addr add 192.168.100.1/24 dev k3kbr0 + sudo ip link set k3kbr0 mtu 1400 sudo ip link set k3kbr0 up # NAT outbound so VMs can reach the internet (image pulls etc). @@ -183,10 +191,12 @@ jobs: # One TAP per VM, attached to the bridge. sudo ip tuntap add tap-w1 mode tap sudo ip link set tap-w1 master k3kbr0 + sudo ip link set tap-w1 mtu 1400 sudo ip link set tap-w1 up sudo ip tuntap add tap-w2 mode tap sudo ip link set tap-w2 master k3kbr0 + sudo ip link set tap-w2 mtu 1400 sudo ip link set tap-w2 up - name: Download Base Cloud Image @@ -226,6 +236,7 @@ jobs: ethernets: ens3: dhcp4: false + mtu: 1400 addresses: [${IP}/24] routes: - to: default @@ -299,16 +310,18 @@ jobs: done - name: Join Workers to K3k Control Plane + env: + K3S_WORKER_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 run: | K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) - echo "Registering Worker 1..." + echo "Registering Worker 1 (k3s ${K3S_WORKER_VERSION})..." ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \ - "curl -sfL https://get.k3s.io | K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -" + "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_WORKER_VERSION} K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -" - echo "Registering Worker 2..." + echo "Registering Worker 2 (k3s ${K3S_WORKER_VERSION})..." ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \ - "curl -sfL https://get.k3s.io | K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -" + "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_WORKER_VERSION} K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -" - name: Verify Cluster Nodes env: From 4fa0e4cd0a151fe9d96d9f3a504c82f21894a759 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Sat, 13 Jun 2026 01:23:52 +0200 Subject: [PATCH 18/36] added debug logs --- .github/workflows/test-conformance-hcp.yaml | 123 ++++++++++++++++++-- 1 file changed, 112 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index 7d6951e2..5c352afe 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -170,16 +170,8 @@ jobs: # Create a bridge so both VMs share an L2 segment with unique routable IPs. # Required because QEMU `-net user` gives every VM the same 10.0.2.15 NAT # address, which breaks flannel VXLAN between workers. - # - # MTU is set to 1400 end-to-end (bridge, TAPs, VM ens3 via cloud-init). - # The path traverses flannel VXLAN inside the virtual cluster AND the - # host's k3s pod network — each strips ~50 bytes for VXLAN headers. - # Defaulting to 1500 across the stack leaves no headroom and PMTUD - # through nested NAT/forwarding is unreliable; 1400 fits both layers - # without fragmentation. sudo ip link add name k3kbr0 type bridge sudo ip addr add 192.168.100.1/24 dev k3kbr0 - sudo ip link set k3kbr0 mtu 1400 sudo ip link set k3kbr0 up # NAT outbound so VMs can reach the internet (image pulls etc). @@ -191,12 +183,10 @@ jobs: # One TAP per VM, attached to the bridge. sudo ip tuntap add tap-w1 mode tap sudo ip link set tap-w1 master k3kbr0 - sudo ip link set tap-w1 mtu 1400 sudo ip link set tap-w1 up sudo ip tuntap add tap-w2 mode tap sudo ip link set tap-w2 master k3kbr0 - sudo ip link set tap-w2 mtu 1400 sudo ip link set tap-w2 up - name: Download Base Cloud Image @@ -236,7 +226,6 @@ jobs: ethernets: ens3: dhcp4: false - mtu: 1400 addresses: [${IP}/24] routes: - to: default @@ -340,6 +329,69 @@ jobs: done ' + # ===================================================================== + # TODO(debug): remove this step once the HCP ExternalName conformance + # failures are root-caused. It reproduces the failing scenario manually + # before the conformance run, so the actual stderr / exit codes show up + # in the workflow output (hydrophone swallows them). + # ===================================================================== + - name: Manual repro probe (before conformance) + env: + KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml + run: | + # Try to reproduce the failing scenario manually so we capture the + # actual stderr from the failing exec, which hydrophone swallows. + # If this fails with rc=139 the same way, we have the bug isolated to + # a 10-line shell script and can iterate on it instead of full conformance. + set +e + kubectl create ns repro + kubectl -n repro create deployment backend \ + --image=registry.k8s.io/e2e-test-images/agnhost:2.59 --replicas=2 \ + -- /agnhost serve-hostname + kubectl -n repro wait deployment/backend --for=condition=Available --timeout=2m + kubectl -n repro expose deployment backend \ + --name=externalsvc --port=80 --target-port=9376 --type=ClusterIP + + kubectl -n repro create deployment echo \ + --image=registry.k8s.io/e2e-test-images/agnhost:2.59 --replicas=2 \ + -- /agnhost serve-hostname + kubectl -n repro wait deployment/echo --for=condition=Available --timeout=2m + kubectl -n repro expose deployment echo \ + --name=svc-a --type=ClusterIP --port=80 --target-port=9376 + + kubectl -n repro patch svc svc-a --type=merge -p '{ + "spec":{"type":"ExternalName","externalName":"externalsvc.repro.svc.cluster.local","clusterIP":"","clusterIPs":null,"ports":null,"selector":null} + }' + + # Mirror the conformance pattern: brand-new exec pod, exec immediately. + kubectl -n repro run execpod \ + --image=registry.k8s.io/e2e-test-images/agnhost:2.59 \ + --command -- sleep infinity + kubectl -n repro wait pod/execpod --for=condition=Ready --timeout=2m + + echo "=== exec #1 (immediate, mirrors conformance) ===" + kubectl -n repro exec execpod -- /bin/sh -x -c "nslookup svc-a.repro.svc.cluster.local"; echo "exit: $?" + + echo "=== exec #2 (after 5s sleep) ===" + sleep 5 + kubectl -n repro exec execpod -- /bin/sh -x -c "nslookup svc-a.repro.svc.cluster.local"; echo "exit: $?" + + echo "=== exec #3 (getent — different resolver) ===" + kubectl -n repro exec execpod -- /bin/sh -c "getent hosts svc-a.repro.svc.cluster.local"; echo "exit: $?" + + echo "=== exec #4 (control: no DNS, just hostname) ===" + kubectl -n repro exec execpod -- /bin/sh -c "cat /etc/hostname"; echo "exit: $?" + + echo "=== /etc/resolv.conf in pod ===" + kubectl -n repro exec execpod -- cat /etc/resolv.conf + + echo "=== pod / node placement ===" + kubectl -n repro get pods -o wide + + # Leave the namespace alive so the conformance run can inspect state later if needed. + # (Cleanup is automatic when the runner tears down.) + true + - name: Run conformance tests run: | hydrophone --focus 'should be able to change the type from (ClusterIP|NodePort) to ExternalName' \ @@ -360,6 +412,36 @@ jobs: journalctl -u k3s -o cat --no-pager > /tmp/k3s.log kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log + # ===================================================================== + # TODO(debug): remove this step once the HCP ExternalName conformance + # failures are root-caused. It exists only to collect extra diagnostics + # (virtual cluster apiserver, worker journals, bridge counters) that the + # standard "Collect logs" step doesn't capture. + # ===================================================================== + - name: Collect debug logs + if: always() + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + run: | + # K3k SERVER POD — runs the virtual cluster's kube-apiserver/kcm/scheduler. + # All exec streams flow through this; most likely place to see the real error. + kubectl logs -n k3k-mycluster k3k-mycluster-server-0 --tail=-1 > /tmp/k3k-server.log || true + kubectl logs -n k3k-mycluster k3k-mycluster-server-0 --previous --tail=-1 > /tmp/k3k-server-prev.log 2>/dev/null || true + + # Worker VM journals — k3s agent + kubelet + containerd live here. + # Captured via SSH because the workers aren't kubectl-accessible. + for i in 1 2; do + IP="192.168.100.1${i}" + ssh -i ${{ github.workspace }}/id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ + "sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}-k3s-agent.log 2>&1 || true + ssh -i ${{ github.workspace }}/id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ + "sudo dmesg --no-pager" > /tmp/worker-${i}-dmesg.log 2>&1 || true + done + + # Bridge / iface counters — if any TX/RX errors or drops grew, MTU/queue is suspect. + ip -s link show k3kbr0 tap-w1 tap-w2 > /tmp/host-iface-stats.log 2>&1 || true + sudo conntrack -L 2>/dev/null | wc -l > /tmp/host-conntrack-count.log || true + - name: Archive K3s logs uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -374,6 +456,25 @@ jobs: name: k3k-${{ matrix.k8s_version }}-logs path: /tmp/k3k.log + # ===================================================================== + # TODO(debug): remove this step alongside "Collect debug logs" once the + # HCP ExternalName conformance failures are root-caused. + # ===================================================================== + - name: Archive debug logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: debug-${{ matrix.k8s_version }}-logs + path: | + /tmp/k3k-server.log + /tmp/k3k-server-prev.log + /tmp/worker-1-k3s-agent.log + /tmp/worker-1-dmesg.log + /tmp/worker-2-k3s-agent.log + /tmp/worker-2-dmesg.log + /tmp/host-iface-stats.log + /tmp/host-conntrack-count.log + - name: Archive conformance logs uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() From 54c21424ee35d1f9b59c99d1e9f0847c79070805 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Sat, 13 Jun 2026 01:56:08 +0200 Subject: [PATCH 19/36] added logs --- .github/workflows/test-conformance-hcp.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index 5c352afe..46e3d747 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -339,6 +339,9 @@ jobs: env: KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml run: | + # Capture everything to a file too, so the result is in the archived + # artifacts (not only in the GHA web viewer). + exec > >(tee /tmp/manual-probe.log) 2>&1 # Try to reproduce the failing scenario manually so we capture the # actual stderr from the failing exec, which hydrophone swallows. # If this fails with rc=139 the same way, we have the bug isolated to @@ -435,11 +438,16 @@ jobs: ssh -i ${{ github.workspace }}/id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ "sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}-k3s-agent.log 2>&1 || true ssh -i ${{ github.workspace }}/id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ - "sudo dmesg --no-pager" > /tmp/worker-${i}-dmesg.log 2>&1 || true + "sudo dmesg -T" > /tmp/worker-${i}-dmesg.log 2>&1 || true done # Bridge / iface counters — if any TX/RX errors or drops grew, MTU/queue is suspect. - ip -s link show k3kbr0 tap-w1 tap-w2 > /tmp/host-iface-stats.log 2>&1 || true + # iproute2 wants `dev` qualifier per-interface; loop instead of listing multiple. + for iface in k3kbr0 tap-w1 tap-w2; do + echo "=== $iface ===" + ip -s link show dev $iface 2>&1 || echo "(missing)" + done > /tmp/host-iface-stats.log + sudo conntrack -L 2>/dev/null | wc -l > /tmp/host-conntrack-count.log || true - name: Archive K3s logs @@ -474,6 +482,7 @@ jobs: /tmp/worker-2-dmesg.log /tmp/host-iface-stats.log /tmp/host-conntrack-count.log + /tmp/manual-probe.log - name: Archive conformance logs uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 From 8091c0ab410669b02f787d41bef9630b7c02f9e1 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Sat, 13 Jun 2026 02:19:26 +0200 Subject: [PATCH 20/36] use 24.04 ubuntu --- .github/workflows/test-conformance-hcp.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index 46e3d747..7d0b499f 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -191,7 +191,12 @@ jobs: - name: Download Base Cloud Image run: | - wget -q https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img + # Use Ubuntu 24.04 (noble) LTS rather than 26.04 (resolute). Resolute's + # cri-containerd AppArmor profile blocks inter-thread signals that the + # BIND ISC library uses during nslookup shutdown — causing every + # conformance ExternalName test that calls `nslookup` from an exec pod + # to exit 139 with "kill: Permission denied". + wget -q https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img -O ubuntu-cloudimg.img - name: Generate SSH Key and Cloud-Init Seeds run: | @@ -240,8 +245,8 @@ jobs: - name: Create Worker Disks run: | - qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G - qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-2.qcow2 20G + qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-1.qcow2 20G + qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-2.qcow2 20G - name: Launch Worker VMs run: | From 7bd41594292cff590ef65dc33de536bc7ce8ee94 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Sat, 13 Jun 2026 02:36:01 +0200 Subject: [PATCH 21/36] removed logs --- .github/workflows/test-conformance-hcp.yaml | 149 +++----------------- 1 file changed, 16 insertions(+), 133 deletions(-) diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index 7d0b499f..ec303517 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -191,11 +191,21 @@ jobs: - name: Download Base Cloud Image run: | - # Use Ubuntu 24.04 (noble) LTS rather than 26.04 (resolute). Resolute's - # cri-containerd AppArmor profile blocks inter-thread signals that the - # BIND ISC library uses during nslookup shutdown — causing every - # conformance ExternalName test that calls `nslookup` from an exec pod - # to exit 139 with "kill: Permission denied". + # PINNED TO UBUNTU 24.04 LTS (noble). + # + # Newer Ubuntu releases (tested: 26.04 "resolute") ship a stricter + # `cri-containerd.apparmor.d` profile that denies inter-thread signal + # delivery. The BIND ISC library used by `nslookup` relies on those + # signals during shutdown (`isc_app_ctxshutdown()` calls `kill()`), + # so when AppArmor denies them nslookup exits 139 with + # "kill: Permission denied". The conformance tests + # `[sig-network] Services should be able to change the type from + # {NodePort,ClusterIP} to ExternalName` + # both run `nslookup` from an exec pod and fail in that case. + # + # Before bumping past 24.04, verify those two conformance tests still + # pass — or that the containerd AppArmor profile on the newer release + # has been relaxed to allow intra-pod signals. wget -q https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img -O ubuntu-cloudimg.img - name: Generate SSH Key and Cloud-Init Seeds @@ -334,84 +344,12 @@ jobs: done ' - # ===================================================================== - # TODO(debug): remove this step once the HCP ExternalName conformance - # failures are root-caused. It reproduces the failing scenario manually - # before the conformance run, so the actual stderr / exit codes show up - # in the workflow output (hydrophone swallows them). - # ===================================================================== - - name: Manual repro probe (before conformance) - env: - KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml - run: | - # Capture everything to a file too, so the result is in the archived - # artifacts (not only in the GHA web viewer). - exec > >(tee /tmp/manual-probe.log) 2>&1 - # Try to reproduce the failing scenario manually so we capture the - # actual stderr from the failing exec, which hydrophone swallows. - # If this fails with rc=139 the same way, we have the bug isolated to - # a 10-line shell script and can iterate on it instead of full conformance. - set +e - kubectl create ns repro - kubectl -n repro create deployment backend \ - --image=registry.k8s.io/e2e-test-images/agnhost:2.59 --replicas=2 \ - -- /agnhost serve-hostname - kubectl -n repro wait deployment/backend --for=condition=Available --timeout=2m - kubectl -n repro expose deployment backend \ - --name=externalsvc --port=80 --target-port=9376 --type=ClusterIP - - kubectl -n repro create deployment echo \ - --image=registry.k8s.io/e2e-test-images/agnhost:2.59 --replicas=2 \ - -- /agnhost serve-hostname - kubectl -n repro wait deployment/echo --for=condition=Available --timeout=2m - kubectl -n repro expose deployment echo \ - --name=svc-a --type=ClusterIP --port=80 --target-port=9376 - - kubectl -n repro patch svc svc-a --type=merge -p '{ - "spec":{"type":"ExternalName","externalName":"externalsvc.repro.svc.cluster.local","clusterIP":"","clusterIPs":null,"ports":null,"selector":null} - }' - - # Mirror the conformance pattern: brand-new exec pod, exec immediately. - kubectl -n repro run execpod \ - --image=registry.k8s.io/e2e-test-images/agnhost:2.59 \ - --command -- sleep infinity - kubectl -n repro wait pod/execpod --for=condition=Ready --timeout=2m - - echo "=== exec #1 (immediate, mirrors conformance) ===" - kubectl -n repro exec execpod -- /bin/sh -x -c "nslookup svc-a.repro.svc.cluster.local"; echo "exit: $?" - - echo "=== exec #2 (after 5s sleep) ===" - sleep 5 - kubectl -n repro exec execpod -- /bin/sh -x -c "nslookup svc-a.repro.svc.cluster.local"; echo "exit: $?" - - echo "=== exec #3 (getent — different resolver) ===" - kubectl -n repro exec execpod -- /bin/sh -c "getent hosts svc-a.repro.svc.cluster.local"; echo "exit: $?" - - echo "=== exec #4 (control: no DNS, just hostname) ===" - kubectl -n repro exec execpod -- /bin/sh -c "cat /etc/hostname"; echo "exit: $?" - - echo "=== /etc/resolv.conf in pod ===" - kubectl -n repro exec execpod -- cat /etc/resolv.conf - - echo "=== pod / node placement ===" - kubectl -n repro get pods -o wide - - # Leave the namespace alive so the conformance run can inspect state later if needed. - # (Cleanup is automatic when the runner tears down.) - true - - name: Run conformance tests run: | - hydrophone --focus 'should be able to change the type from (ClusterIP|NodePort) to ExternalName' \ + hydrophone --conformance --parallel 4 \ --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ --output-dir /tmp - # - name: Run conformance tests - # run: | - # hydrophone --conformance --parallel 4 \ - # --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ - # --output-dir /tmp - - name: Collect logs if: always() env: @@ -420,41 +358,6 @@ jobs: journalctl -u k3s -o cat --no-pager > /tmp/k3s.log kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log - # ===================================================================== - # TODO(debug): remove this step once the HCP ExternalName conformance - # failures are root-caused. It exists only to collect extra diagnostics - # (virtual cluster apiserver, worker journals, bridge counters) that the - # standard "Collect logs" step doesn't capture. - # ===================================================================== - - name: Collect debug logs - if: always() - env: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml - run: | - # K3k SERVER POD — runs the virtual cluster's kube-apiserver/kcm/scheduler. - # All exec streams flow through this; most likely place to see the real error. - kubectl logs -n k3k-mycluster k3k-mycluster-server-0 --tail=-1 > /tmp/k3k-server.log || true - kubectl logs -n k3k-mycluster k3k-mycluster-server-0 --previous --tail=-1 > /tmp/k3k-server-prev.log 2>/dev/null || true - - # Worker VM journals — k3s agent + kubelet + containerd live here. - # Captured via SSH because the workers aren't kubectl-accessible. - for i in 1 2; do - IP="192.168.100.1${i}" - ssh -i ${{ github.workspace }}/id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ - "sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}-k3s-agent.log 2>&1 || true - ssh -i ${{ github.workspace }}/id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ - "sudo dmesg -T" > /tmp/worker-${i}-dmesg.log 2>&1 || true - done - - # Bridge / iface counters — if any TX/RX errors or drops grew, MTU/queue is suspect. - # iproute2 wants `dev` qualifier per-interface; loop instead of listing multiple. - for iface in k3kbr0 tap-w1 tap-w2; do - echo "=== $iface ===" - ip -s link show dev $iface 2>&1 || echo "(missing)" - done > /tmp/host-iface-stats.log - - sudo conntrack -L 2>/dev/null | wc -l > /tmp/host-conntrack-count.log || true - - name: Archive K3s logs uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -469,26 +372,6 @@ jobs: name: k3k-${{ matrix.k8s_version }}-logs path: /tmp/k3k.log - # ===================================================================== - # TODO(debug): remove this step alongside "Collect debug logs" once the - # HCP ExternalName conformance failures are root-caused. - # ===================================================================== - - name: Archive debug logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: debug-${{ matrix.k8s_version }}-logs - path: | - /tmp/k3k-server.log - /tmp/k3k-server-prev.log - /tmp/worker-1-k3s-agent.log - /tmp/worker-1-dmesg.log - /tmp/worker-2-k3s-agent.log - /tmp/worker-2-dmesg.log - /tmp/host-iface-stats.log - /tmp/host-conntrack-count.log - /tmp/manual-probe.log - - name: Archive conformance logs uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() From 462433b28c7fb31f3b53b3dc06a25bc3b031dbb1 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 15 Jun 2026 14:46:23 +0200 Subject: [PATCH 22/36] added HCP print command --- cli/cmds/cluster_create.go | 40 +++++++++++++++++++++++++++----------- cli/cmds/kubeconfig.go | 32 +++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/cli/cmds/cluster_create.go b/cli/cmds/cluster_create.go index cbfcd654..9978ce2f 100644 --- a/cli/cmds/cluster_create.go +++ b/cli/cmds/cluster_create.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "net/url" "os" "strings" "text/template" @@ -89,7 +88,7 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm } if config.mode == string(v1beta1.HCPClusterMode) { - logrus.Warn("Mode 'hcp' is experimental") + logrus.Warn("HCP (Hosted Control Plane) mode is experimental.") } namespace := appCtx.Namespace(name) @@ -132,17 +131,12 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm } // add Host IP address as an extra TLS-SAN to expose the k3k cluster - url, err := url.Parse(appCtx.RestConfig.Host) + host, err := resolveServerHost(appCtx.RestConfig.Host, config.kubeconfigServerHost) if err != nil { return err } - host := strings.Split(url.Host, ":") - if config.kubeconfigServerHost != "" { - host = []string{config.kubeconfigServerHost} - } - - cluster.Spec.TLSSANs = []string{host[0]} + cluster.Spec.TLSSANs = []string{host} if err := client.Create(ctx, cluster); err != nil { if apierrors.IsAlreadyExists(err) { @@ -183,16 +177,40 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm var kubeconfig *clientcmdapi.Config if err := retry.OnError(availableBackoff, apierrors.IsNotFound, func() error { - kubeconfig, err = cfg.Generate(ctx, client, cluster, host[0], 0) + kubeconfig, err = cfg.Generate(ctx, client, cluster, host, 0) return err }); err != nil { return err } - return writeKubeconfigFile(cluster, kubeconfig, "") + if err := writeKubeconfigFile(cluster, kubeconfig, ""); err != nil { + return err + } + + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + printHCPJoinInstructions(cluster, kubeconfig) + } + + return nil } } +func printHCPJoinInstructions(cluster *v1beta1.Cluster, kc *clientcmdapi.Config) { + tokenSecretName := k3kcluster.TokenSecretName(cluster.Name) + serverURL := kc.Clusters["default"].Server + + logrus.Infof(`To join an external worker node to this HCP cluster: + + 1. On this machine, fetch the cluster token: + + kubectl get secret -n %s %s -o jsonpath='{.data.token}' | base64 -d + + 2. On the worker node, run (replace with the value from step 1): + + curl -sfL https://get.k3s.io | K3S_URL=%s K3S_TOKEN= sh - +`, cluster.Namespace, tokenSecretName, serverURL) +} + func newCluster(name, namespace string, config *CreateConfig) (*v1beta1.Cluster, error) { var storageRequestSize *resource.Quantity if config.storageRequestSize != "" { diff --git a/cli/cmds/kubeconfig.go b/cli/cmds/kubeconfig.go index 39763fd4..1b3ad1b5 100644 --- a/cli/cmds/kubeconfig.go +++ b/cli/cmds/kubeconfig.go @@ -89,14 +89,12 @@ func generate(appCtx *AppContext, cfg *GenerateKubeconfigConfig) func(cmd *cobra return err } - url, err := url.Parse(appCtx.RestConfig.Host) + host, err := resolveServerHost(appCtx.RestConfig.Host, cfg.kubeconfigServerHost) if err != nil { return err } - host := strings.Split(url.Host, ":") if cfg.kubeconfigServerHost != "" { - host = []string{cfg.kubeconfigServerHost} cfg.altNames = append(cfg.altNames, cfg.kubeconfigServerHost) } @@ -118,16 +116,40 @@ func generate(appCtx *AppContext, cfg *GenerateKubeconfigConfig) func(cmd *cobra var kubeconfig *clientcmdapi.Config if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error { - kubeconfig, err = kubeCfg.Generate(ctx, client, &cluster, host[0], 0) + kubeconfig, err = kubeCfg.Generate(ctx, client, &cluster, host, 0) return err }); err != nil { return err } - return writeKubeconfigFile(&cluster, kubeconfig, cfg.configName) + if err := writeKubeconfigFile(&cluster, kubeconfig, cfg.configName); err != nil { + return err + } + + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + printHCPJoinInstructions(&cluster, kubeconfig) + } + + return nil } } +// resolveServerHost returns the host that should be embedded in the kubeconfig +// server URL and used as the TLS-SAN. If override is set it takes precedence; +// otherwise the host is extracted from restConfigHost. +func resolveServerHost(restConfigHost, override string) (string, error) { + if override != "" { + return override, nil + } + + u, err := url.Parse(restConfigHost) + if err != nil { + return "", err + } + + return strings.Split(u.Host, ":")[0], nil +} + func writeKubeconfigFile(cluster *v1beta1.Cluster, kubeconfig *clientcmdapi.Config, configName string) error { if configName == "" { configName = cluster.Namespace + "-" + cluster.Name + "-kubeconfig.yaml" From 051f8ddfc1907cf04dc6ccbc129d00d1b7513c82 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 15 Jun 2026 15:02:47 +0200 Subject: [PATCH 23/36] removed hcp-test from Tests --- .github/workflows/test.yaml | 265 ------------------------------------ 1 file changed, 265 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b9a288b9..23d9d5f4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,273 +12,8 @@ permissions: env: KUBERNETES_VERSION: v1.35.3 - HELM_VERSION: v4.1.3 - HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700 jobs: - hcp-test: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - fetch-depth: 0 - fetch-tags: true - - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 - with: - go-version-file: go.mod - - - name: Install helm - env: - FILENAME: helm.tar.gz - run: | - curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz - echo "${{ env.HELM_CHECKSUM_AMD64 }} ${{ env.FILENAME }}" | sha256sum --check - tar -xvzf ${{ env.FILENAME }} linux-amd64/helm - sudo install -m 755 linux-amd64/helm /usr/local/bin/helm - - rm -fr "${{ env.FILENAME }}" linux-amd64/helm - - - name: Install k3s - env: - K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 - run: | - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - - - export KUBECONFIG=/etc/rancher/k3s/k3s.yaml - echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV - - kubectl cluster-info - kubectl get nodes - - - name: Setup K3k (from source) - run: | - export REPO=ttl.sh/$(uuidgen) - export VERSION=1h - - make build - make package - make push - make install - - # add k3kcli to $PATH - echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - - - name: Wait for K3k controller - run: | - echo "Wait for K3k controller deployment to be available" - kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m - - - name: Check k3kcli - run: k3kcli -v - - - name: Create virtual cluster - run: | - kubectl create namespace k3k-mycluster - - cat < user-data - #cloud-config - users: - - name: ubuntu - ssh_authorized_keys: - - $(cat ./id_rsa.pub) - sudo: ['ALL=(ALL) NOPASSWD:ALL'] - shell: /bin/bash - EOF - - # Bake the configuration into a reusable metadata ISO - cloud-localds seed.img user-data - - # 4. Provision independent overlay disks for Worker - - name: Create Worker Disks - run: | - qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G - - # 5. Boot both VMs headlessly in the background with unique MACs and forwarded SSH ports - - name: Launch Worker VM - run: | - # Launch Worker 1 (SSH forwarded to Host Port 2222) - sudo qemu-system-x86_64 \ - -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ - -drive file=worker-1.qcow2,if=virtio \ - -drive file=seed.img,format=raw,if=virtio \ - -net nic,model=virtio,macaddr=52:54:00:12:34:56 \ - -net user,hostfwd=tcp::2222-:22 \ - & - - # 6. Block the script until both VMs are completely booted and listening on SSH - - name: Wait for SSH Availability - run: | - echo "Waiting for Worker (Port 2222) to respond..." - timeout 120s bash -c ' - until ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done - ' - - echo "VM is completely up and accessible!" - - - - name: Join Worker to K3k Control Plane - run: | - K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) - - echo "Testing connectivity from VM to K3k API server..." - ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'VM connectivity test failed'" - - echo "Registering Worker..." - set +e # Don't exit on error, we want to collect logs - ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 bash -s </dev/null | grep -c "Ready") -eq 1 ]; do - echo "Waiting for both worker nodes to show Ready..." - kubectl get nodes || true - sleep 5 - done - ' - - - name: Collect logs - if: always() - run: | - journalctl -u k3s -o cat --no-pager > /tmp/k3s.log - kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log - - - name: Archive K3s logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: k3s-hcp-logs - path: /tmp/k3s.log - - - name: Archive K3k logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: k3k-hcp-logs - path: /tmp/k3k.log - tests: runs-on: ubuntu-latest From cc61345ddabaddd9d2e57b87a33b47b92dc75735 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 15 Jun 2026 15:11:11 +0200 Subject: [PATCH 24/36] fixed template and comments --- pkg/controller/cluster/server/config.go | 38 +++++++---------------- pkg/controller/cluster/server/template.go | 2 +- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index 31a30549..c24279d1 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -85,39 +85,23 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"} case v1beta1.HCPClusterMode: serverConfig.DisableAgent = true + // Tunnel apiserver egress through the k3s-agent WebSocket: the + // apiserver has no route to the virtual cluster's pod CIDR and + // bypasses kube-proxy when dialing pod IPs (webhooks, log/exec). + // "cluster" is the only safe mode — "agent" lets pod dials go + // direct (no route, fails); "pod" only permits pod IPs the agent + // has already watched, so a newly-created pod's IP is rejected + // and tears down the remotedialer session, making kubelet streams + // flaky. See k3s pkg/agent/tunnel/tunnel.go. serverConfig.EgressSelectorMode = "cluster" - // Disable it so K3k can own that Endpoints object and point - // it at the externally-reachable host:port (NodePort / LB / Ingress). + // Disable the apiserver's built-in endpoint reconciler so K3k can + // own default/kubernetes Endpoints and point it at the externally + // reachable host:port (NodePort / LB / Ingress). serverConfig.KubeApiServerArg = append(serverConfig.KubeApiServerArg, "endpoint-reconciler-type=none") case v1beta1.VirtualClusterMode: // no extra config for virtual mode } - // In shared mode workloads run on the host cluster, so the apiserver pod - // can reach them directly via the host pod network and the egress - // selector is unnecessary. - // - // In hcp mode the apiserver pod has NO route to the virtual cluster's - // pod CIDR (which only exists on joined external worker nodes), and the - // kube-apiserver bypasses kube-proxy when calling webhooks / proxying - // to pods: it resolves Service -> Endpoints itself and dials the Pod IP - // directly. We therefore tunnel apiserver egress through the WebSocket - // each k3s-agent maintains back to the server. - // - // We pick "cluster" rather than "pod" or "agent" because the agent-side - // authorizer differs by mode (k3s pkg/agent/tunnel/tunnel.go): - // - agent: only kubelet calls are tunneled; pod-IP dials go direct - // and fail in HCP (no route to virtual pod CIDR). - // - pod: authorizer only allows pod IPs the agent has *already - // watched*. A newly-created pod's IP is rejected with - // "connect not allowed", which terminates the entire - // remotedialer session and 502s in-flight kubelet streams - // -> kubectl logs / exec / webhooks become flaky. - // - cluster: authorizer pre-populates the cluster CIDR + node IPs as - // non-hostNet entries, so every pod IP and every node port - // is permitted. No race, no per-port allowlist. This is - // what we want for a managed control plane. - return serverConfig } diff --git a/pkg/controller/cluster/server/template.go b/pkg/controller/cluster/server/template.go index 774fb985..0fd48d4e 100644 --- a/pkg/controller/cluster/server/template.go +++ b/pkg/controller/cluster/server/template.go @@ -19,7 +19,7 @@ safe_mode() { CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip) fi - if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" = "shared" ] || [ "{{.K3K_MODE}}" = "hcp" ]; then + if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then return fi From 3cd0ea5775207cb928330529dc7f6b2112d1abfa Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 15 Jun 2026 15:11:28 +0200 Subject: [PATCH 25/36] uncommented generate --- scripts/generate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate b/scripts/generate index 5b618ca4..820d958f 100755 --- a/scripts/generate +++ b/scripts/generate @@ -15,5 +15,5 @@ go run sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSIO # add the 'helm.sh/resource-policy: keep' annotation to the CRDs for f in ./charts/k3k/templates/crds/*.yaml; do echo "Annotating $f" - #yq -c -i '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$f" + yq -c -i '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$f" done From fcb0baeac920553ceb3cb69caeb892716afc4c1e Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 15 Jun 2026 15:13:55 +0200 Subject: [PATCH 26/36] updated crds --- .../k3k/templates/crds/k3k.io_clusters.yaml | 638 ++++++------------ .../crds/k3k.io_virtualclusterpolicies.yaml | 297 +++----- 2 files changed, 297 insertions(+), 638 deletions(-) diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index ca528890..80d62d19 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -4,6 +4,7 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep name: clusters.k3k.io spec: group: k3k.io @@ -54,11 +55,9 @@ spec: description: Spec defines the desired state of the Cluster. properties: addons: - description: Addons specifies secrets containing raw YAML to deploy - on cluster startup. + description: Addons specifies secrets containing raw YAML to deploy on cluster startup. items: - description: Addon specifies a Secret containing YAML to be deployed - on cluster startup. + description: Addon specifies a Secret containing YAML to be deployed on cluster startup. properties: secretNamespace: description: SecretNamespace is the namespace of the Secret. @@ -74,8 +73,7 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the - pod. + description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -94,20 +92,17 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the - corresponding weight. + description: A node selector term, associated with the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -132,16 +127,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -168,8 +161,7 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding - nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -187,8 +179,7 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. + description: Required. A list of node selector terms. The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -196,16 +187,14 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -230,16 +219,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -273,8 +260,7 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate - this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -288,12 +274,10 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -301,17 +285,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -381,17 +362,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -481,16 +459,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -560,16 +536,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -627,9 +601,7 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. - avoid putting this pod in the same node, zone, etc. as some - other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -643,12 +615,10 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -656,17 +626,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -736,17 +703,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -836,16 +800,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -915,16 +877,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -990,11 +950,9 @@ spec: type: string type: array agentEnvs: - description: AgentEnvs specifies list of environment variables to - set in the agent pod. + description: AgentEnvs specifies list of environment variables to set in the agent pod. items: - description: EnvVar represents an environment variable present in - a Container. + description: EnvVar represents an environment variable present in a Container. properties: name: description: |- @@ -1014,8 +972,7 @@ spec: Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot - be used if value is not empty. + description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -1033,8 +990,7 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key - must be defined + description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key @@ -1046,12 +1002,10 @@ spec: spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is - written in terms of, defaults to "v1". + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified - API version. + description: Path of the field to select in the specified API version. type: string required: - fieldPath @@ -1085,8 +1039,7 @@ spec: Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing - the env file. + description: The name of the volume mount containing the env file. type: string required: - key @@ -1100,15 +1053,13 @@ spec: (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, - optional for env vars' + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed - resources, defaults to "1" + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -1122,8 +1073,7 @@ spec: description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must - be a valid secret key. + description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" @@ -1135,8 +1085,7 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must - be defined + description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -1177,16 +1126,14 @@ spec: - message: clusterDNS is immutable rule: self == oldSelf customCAs: - description: CustomCAs specifies the cert/key pairs for custom CA - certificates. + description: CustomCAs specifies the cert/key pairs for custom CA certificates. properties: enabled: default: true description: Enabled toggles this feature on or off. type: boolean sources: - description: Sources defines the sources for all required custom - CA certificates. + description: Sources defines the sources for all required custom CA certificates. properties: clientCA: description: ClientCA specifies the client-ca cert/key pair. @@ -1201,8 +1148,7 @@ spec: - secretName type: object etcdPeerCA: - description: ETCDPeerCA specifies the etcd-peer-ca cert/key - pair. + description: ETCDPeerCA specifies the etcd-peer-ca cert/key pair. properties: secretName: description: |- @@ -1214,8 +1160,7 @@ spec: - secretName type: object etcdServerCA: - description: ETCDServerCA specifies the etcd-server-ca cert/key - pair. + description: ETCDServerCA specifies the etcd-server-ca cert/key pair. properties: secretName: description: |- @@ -1227,8 +1172,7 @@ spec: - secretName type: object requestHeaderCA: - description: RequestHeaderCA specifies the request-header-ca - cert/key pair. + description: RequestHeaderCA specifies the request-header-ca cert/key pair. properties: secretName: description: |- @@ -1252,8 +1196,7 @@ spec: - secretName type: object serviceAccountToken: - description: ServiceAccountToken specifies the service-account-token - key. + description: ServiceAccountToken specifies the service-account-token key. properties: secretName: description: |- @@ -1282,23 +1225,19 @@ spec: By default, it's only exposed as a ClusterIP. properties: ingress: - description: Ingress specifies options for exposing the API server - through an Ingress. + description: Ingress specifies options for exposing the API server through an Ingress. properties: annotations: additionalProperties: type: string - description: Annotations specifies annotations to add to the - Ingress. + description: Annotations specifies annotations to add to the Ingress. type: object ingressClassName: - description: IngressClassName specifies the IngressClass to - use for the Ingress. + description: IngressClassName specifies the IngressClass to use for the Ingress. type: string type: object loadBalancer: - description: LoadBalancer specifies options for exposing the API - server through a LoadBalancer service. + description: LoadBalancer specifies options for exposing the API server through a LoadBalancer service. properties: etcdPort: description: |- @@ -1316,8 +1255,7 @@ spec: type: integer type: object nodePort: - description: NodePort specifies options for exposing the API server - through NodePort. + description: NodePort specifies options for exposing the API server through NodePort. properties: etcdPort: description: |- @@ -1336,10 +1274,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: ingress, loadbalancer and nodePort are mutually exclusive; - only one can be set - rule: '[has(self.ingress), has(self.loadBalancer), has(self.nodePort)].filter(x, - x).size() <= 1' + - message: ingress, loadbalancer and nodePort are mutually exclusive; only one can be set + rule: '[has(self.ingress), has(self.loadBalancer), has(self.nodePort)].filter(x, x).size() <= 1' hostUsers: description: |- HostUsers sets the user namespace for server and agent pods. @@ -1480,8 +1416,7 @@ spec: secret contents will be mounted. type: string optional: - description: optional field specify whether the Secret or its - keys must be defined + description: optional field specify whether the Secret or its keys must be defined type: boolean role: description: |- @@ -1623,20 +1558,16 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to - the container. + description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies to - the container. + description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies to - the container. + description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies to - the container. + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: @@ -1679,8 +1610,7 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA - credential spec to use. + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: description: |- @@ -1704,8 +1634,7 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the - pod. + description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1724,20 +1653,17 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the - corresponding weight. + description: A node selector term, associated with the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -1762,16 +1688,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -1798,8 +1722,7 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding - nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -1817,8 +1740,7 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. + description: Required. A list of node selector terms. The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -1826,16 +1748,14 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -1860,16 +1780,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -1903,8 +1821,7 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate - this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1918,12 +1835,10 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -1931,17 +1846,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -2011,17 +1923,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -2111,16 +2020,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -2190,16 +2097,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -2257,9 +2162,7 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. - avoid putting this pod in the same node, zone, etc. as some - other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -2273,12 +2176,10 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -2286,17 +2187,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -2366,17 +2264,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -2466,16 +2361,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -2545,16 +2438,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -2620,11 +2511,9 @@ spec: type: string type: array serverEnvs: - description: ServerEnvs specifies list of environment variables to - set in the server pod. + description: ServerEnvs specifies list of environment variables to set in the server pod. items: - description: EnvVar represents an environment variable present in - a Container. + description: EnvVar represents an environment variable present in a Container. properties: name: description: |- @@ -2644,8 +2533,7 @@ spec: Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot - be used if value is not empty. + description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -2663,8 +2551,7 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key - must be defined + description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key @@ -2676,12 +2563,10 @@ spec: spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is - written in terms of, defaults to "v1". + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified - API version. + description: Path of the field to select in the specified API version. type: string required: - fieldPath @@ -2715,8 +2600,7 @@ spec: Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing - the env file. + description: The name of the volume mount containing the env file. type: string required: - key @@ -2730,15 +2614,13 @@ spec: (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, - optional for env vars' + description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed - resources, defaults to "1" + description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -2752,8 +2634,7 @@ spec: description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must - be a valid secret key. + description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" @@ -2765,8 +2646,7 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must - be defined + description: Specify whether the Secret or its key must be defined type: boolean required: - key @@ -2787,8 +2667,7 @@ spec: description: ServerLimit specifies resource limits for server nodes. type: object serverResources: - description: ServerResources specifies resources limits and requests - for server nodes. + description: ServerResources specifies resources limits and requests for server nodes. properties: claims: description: |- @@ -2867,8 +2746,7 @@ spec: rule: self == oldSelf sync: default: {} - description: Sync specifies the resources types that will be synced - from virtual cluster to host cluster. + description: Sync specifies the resources types that will be synced from virtual cluster to host cluster. properties: configMaps: default: @@ -3009,8 +2887,7 @@ spec: type: object type: object tlsSANs: - description: TLSSANs specifies subject alternative names for the K3s - server certificate. + description: TLSSANs specifies subject alternative names for the K3s server certificate. items: type: string type: array @@ -3020,12 +2897,10 @@ spec: The Secret must have a "token" field in its data. properties: name: - description: name is unique within a namespace to reference a - secret resource. + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: namespace defines the space within which the secret - name must be unique. + description: namespace defines the space within which the secret name must be unique. type: string type: object x-kubernetes-map-type: atomic @@ -3045,8 +2920,7 @@ spec: description: WorkerLimit specifies resource limits for agent nodes. type: object workerResources: - description: WorkerResources specifies resources limits and requests - for worker nodes. + description: WorkerResources specifies resources limits and requests for worker nodes. properties: claims: description: |- @@ -3116,11 +2990,9 @@ spec: description: ClusterDNS is the IP address for the CoreDNS service. type: string conditions: - description: Conditions are the individual conditions for the cluster - set. + description: Conditions are the individual conditions for the cluster set. items: - description: Condition contains details for one aspect of the current - state of this API Resource. + description: Condition contains details for one aspect of the current state of this API Resource. properties: lastTransitionTime: description: |- @@ -3177,13 +3049,11 @@ spec: description: HostVersion is the Kubernetes version of the host node. type: string kubeletPort: - description: KubeletPort specefies the port used by k3k-kubelet in - shared mode. + description: KubeletPort specefies the port used by k3k-kubelet in shared mode. type: integer phase: default: Unknown - description: Phase is a high-level summary of the cluster's current - lifecycle state. + description: Phase is a high-level summary of the cluster's current lifecycle state. enum: - Pending - Provisioning @@ -3203,8 +3073,7 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for - the pod. + description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3223,20 +3092,17 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with - the corresponding weight. + description: A node selector term, associated with the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -3261,16 +3127,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -3297,8 +3161,7 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the - corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -3316,8 +3179,7 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. + description: Required. A list of node selector terms. The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -3325,16 +3187,14 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -3359,16 +3219,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -3402,9 +3260,7 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3418,13 +3274,10 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -3432,17 +3285,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key - that the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -3512,17 +3362,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key - that the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -3612,17 +3459,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -3692,17 +3536,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -3760,9 +3601,7 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, etc. - as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3776,13 +3615,10 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -3790,17 +3626,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key - that the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -3870,17 +3703,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key - that the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -3970,17 +3800,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -4050,17 +3877,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -4126,19 +3950,16 @@ spec: This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. type: boolean name: - description: name is the name of the VirtualClusterPolicy currently - applied to this cluster. + description: name is the name of the VirtualClusterPolicy currently applied to this cluster. minLength: 1 type: string nodeSelector: additionalProperties: type: string - description: nodeSelector is a node selector enforced by the active - VirtualClusterPolicy. + description: nodeSelector is a node selector enforced by the active VirtualClusterPolicy. type: object priorityClass: - description: priorityClass is the priority class enforced by the - active VirtualClusterPolicy. + description: priorityClass is the priority class enforced by the active VirtualClusterPolicy. type: string runtimeClassName: description: |- @@ -4193,16 +4014,14 @@ spec: add: description: Added capabilities items: - description: Capability represent POSIX capabilities - type + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities - type + description: Capability represent POSIX capabilities type type: string type: array x-kubernetes-list-type: atomic @@ -4264,20 +4083,16 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies - to the container. + description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies - to the container. + description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies - to the container. + description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies - to the container. + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: @@ -4320,8 +4135,7 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: description: |- @@ -4345,8 +4159,7 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for - the pod. + description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4365,20 +4178,17 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with - the corresponding weight. + description: A node selector term, associated with the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -4403,16 +4213,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -4439,8 +4247,7 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the - corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -4458,8 +4265,7 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. + description: Required. A list of node selector terms. The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -4467,16 +4273,14 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -4501,16 +4305,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -4544,9 +4346,7 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4560,13 +4360,10 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -4574,17 +4371,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key - that the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -4654,17 +4448,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key - that the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -4754,17 +4545,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -4834,17 +4622,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -4902,9 +4687,7 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, etc. - as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4918,13 +4701,10 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -4932,17 +4712,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key - that the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -5012,17 +4789,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key - that the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -5112,17 +4886,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -5192,17 +4963,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -5405,15 +5173,13 @@ spec: - name type: object policyName: - description: PolicyName specifies the virtual cluster policy name - bound to the virtual cluster. + description: PolicyName specifies the virtual cluster policy name bound to the virtual cluster. type: string serviceCIDR: description: ServiceCIDR is the CIDR range for service IPs. type: string tlsSANs: - description: TLSSANs specifies subject alternative names for the K3s - server certificate. + description: TLSSANs specifies subject alternative names for the K3s server certificate. items: type: string type: array diff --git a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml index 184c5f98..a708cca4 100644 --- a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml +++ b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml @@ -4,6 +4,7 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep name: virtualclusterpolicies.k3k.io spec: group: k3k.io @@ -59,8 +60,7 @@ spec: - virtual - hcp default: shared - description: AllowedMode specifies the allowed cluster provisioning - mode. Defaults to "shared". + description: AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". type: string x-kubernetes-validations: - message: mode is immutable @@ -71,8 +71,7 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the - pod. + description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -91,20 +90,17 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the - corresponding weight. + description: A node selector term, associated with the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -129,16 +125,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -165,8 +159,7 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding - nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -184,8 +177,7 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. + description: Required. A list of node selector terms. The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -193,16 +185,14 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -227,16 +217,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -270,8 +258,7 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate - this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -285,12 +272,10 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -298,17 +283,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -378,17 +360,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -478,16 +457,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -557,16 +534,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -624,9 +599,7 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. - avoid putting this pod in the same node, zone, etc. as some - other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -640,12 +613,10 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -653,17 +624,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -733,17 +701,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -833,16 +798,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -912,16 +875,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -982,12 +943,10 @@ spec: defaultNodeSelector: additionalProperties: type: string - description: DefaultNodeSelector specifies the node selector that - applies to all clusters (server + agent) in the target Namespace. + description: DefaultNodeSelector specifies the node selector that applies to all clusters (server + agent) in the target Namespace. type: object defaultPriorityClass: - description: DefaultPriorityClass specifies the priorityClassName - applied to all pods of all clusters in the target Namespace. + description: DefaultPriorityClass specifies the priorityClassName applied to all pods of all clusters in the target Namespace. type: string defaultServerAffinity: description: |- @@ -995,8 +954,7 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the - pod. + description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1015,20 +973,17 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the - corresponding weight. + description: A node selector term, associated with the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -1053,16 +1008,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -1089,8 +1042,7 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding - nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -1108,8 +1060,7 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. + description: Required. A list of node selector terms. The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -1117,16 +1068,14 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements - by node's labels. + description: A list of node selector requirements by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -1151,16 +1100,14 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements - by node's fields. + description: A list of node selector requirements by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector - applies to. + description: The label key that the selector applies to. type: string operator: description: |- @@ -1194,8 +1141,7 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate - this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1209,12 +1155,10 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -1222,17 +1166,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -1302,17 +1243,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -1402,16 +1340,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -1481,16 +1417,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -1548,9 +1482,7 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. - avoid putting this pod in the same node, zone, etc. as some - other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1564,12 +1496,10 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: |- @@ -1577,17 +1507,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -1657,17 +1584,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that - the selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -1757,16 +1681,14 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -1836,16 +1758,14 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the - selector applies to. + description: key is the label key that the selector applies to. type: string operator: description: |- @@ -1904,8 +1824,7 @@ spec: type: object type: object disableNetworkPolicy: - description: DisableNetworkPolicy indicates whether to disable the - creation of a default network policy for cluster isolation. + description: DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. type: boolean hostUsers: description: |- @@ -1920,11 +1839,9 @@ spec: to set defaults and constraints (min/max) properties: limits: - description: Limits is the list of LimitRangeItem objects that - are enforced. + description: Limits is the list of LimitRangeItem objects that are enforced. items: - description: LimitRangeItem defines a min/max usage limit for - any resource that matches on kind. + description: LimitRangeItem defines a min/max usage limit for any resource that matches on kind. properties: default: additionalProperties: @@ -1933,8 +1850,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Default resource requirement limit value by - resource name if resource limit is omitted. + description: Default resource requirement limit value by resource name if resource limit is omitted. type: object defaultRequest: additionalProperties: @@ -1943,9 +1859,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: DefaultRequest is the default resource requirement - request value by resource name if resource request is - omitted. + description: DefaultRequest is the default resource requirement request value by resource name if resource request is omitted. type: object max: additionalProperties: @@ -1954,8 +1868,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Max usage constraints on this kind by resource - name. + description: Max usage constraints on this kind by resource name. type: object maxLimitRequestRatio: additionalProperties: @@ -1964,11 +1877,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: MaxLimitRequestRatio if specified, the named - resource must have a request and limit that are both non-zero - where limit divided by request is less than or equal to - the enumerated value; this represents the max burst for - the named resource. + description: MaxLimitRequestRatio if specified, the named resource must have a request and limit that are both non-zero where limit divided by request is less than or equal to the enumerated value; this represents the max burst for the named resource. type: object min: additionalProperties: @@ -1977,8 +1886,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Min usage constraints on this kind by resource - name. + description: Min usage constraints on this kind by resource name. type: object type: description: Type of resource that this limit applies to. @@ -1992,16 +1900,14 @@ spec: - limits type: object podSecurityAdmissionLevel: - description: PodSecurityAdmissionLevel specifies the pod security - admission level applied to the pods in the namespace. + description: PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. enum: - privileged - baseline - restricted type: string quota: - description: Quota specifies the resource limits for clusters within - a clusterpolicy. + description: Quota specifies the resource limits for clusters within a clusterpolicy. properties: hard: additionalProperties: @@ -2021,8 +1927,7 @@ spec: For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. properties: matchExpressions: - description: A list of scope selector requirements by scope - of the resources. + description: A list of scope selector requirements by scope of the resources. items: description: |- A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator @@ -2034,8 +1939,7 @@ spec: Valid operators are In, NotIn, Exists, DoesNotExist. type: string scopeName: - description: The name of the scope that the selector - applies to. + description: The name of the scope that the selector applies to. type: string values: description: |- @@ -2060,8 +1964,7 @@ spec: A collection of filters that must match each object tracked by a quota. If not specified, the quota matches all objects. items: - description: A ResourceQuotaScope defines a filter that must - match each object tracked by a quota + description: A ResourceQuotaScope defines a filter that must match each object tracked by a quota type: string type: array x-kubernetes-list-type: atomic @@ -2188,20 +2091,16 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to - the container. + description: Level is SELinux level label that applies to the container. type: string role: - description: Role is a SELinux role label that applies to - the container. + description: Role is a SELinux role label that applies to the container. type: string type: - description: Type is a SELinux type label that applies to - the container. + description: Type is a SELinux type label that applies to the container. type: string user: - description: User is a SELinux user label that applies to - the container. + description: User is a SELinux user label that applies to the container. type: string type: object seccompProfile: @@ -2244,8 +2143,7 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA - credential spec to use. + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string hostProcess: description: |- @@ -2265,8 +2163,7 @@ spec: type: object sync: default: {} - description: Sync specifies the resources types that will be synced - from virtual cluster to host cluster. + description: Sync specifies the resources types that will be synced from virtual cluster to host cluster. properties: configMaps: default: @@ -2411,11 +2308,9 @@ spec: description: Status reflects the observed state of the VirtualClusterPolicy. properties: conditions: - description: Conditions are the individual conditions for the cluster - set. + description: Conditions are the individual conditions for the cluster set. items: - description: Condition contains details for one aspect of the current - state of this API Resource. + description: Condition contains details for one aspect of the current state of this API Resource. properties: lastTransitionTime: description: |- @@ -2469,12 +2364,10 @@ spec: type: object type: array lastUpdateTime: - description: LastUpdate is the timestamp when the status was last - updated. + description: LastUpdate is the timestamp when the status was last updated. type: string observedGeneration: - description: ObservedGeneration was the generation at the time the - status was updated. + description: ObservedGeneration was the generation at the time the status was updated. format: int64 type: integer summary: From d01ae2e8203e75113f7a982a87dde46c2028e932 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 15 Jun 2026 16:03:28 +0200 Subject: [PATCH 27/36] adding e2e tests --- pkg/controller/cluster/hcp.go | 23 ++++------ tests/e2e/cluster_create_test.go | 79 ++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go index 127c5e0c..c6d5f299 100644 --- a/pkg/controller/cluster/hcp.go +++ b/pkg/controller/cluster/hcp.go @@ -151,15 +151,9 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context endpointSlice.Labels[discoveryv1.LabelServiceName] = "kubernetes" endpointSlice.AddressType = addressType - endpoint := discoveryv1.Endpoint{ - Addresses: []string{addr.IP}, + endpointSlice.Endpoints = []discoveryv1.Endpoint{ + {Addresses: []string{addr.IP}}, } - - if addr.Hostname != "" { - endpoint.Hostname = &addr.Hostname - } - - endpointSlice.Endpoints = []discoveryv1.Endpoint{endpoint} portName := "https" endpointSlice.Ports = []discoveryv1.EndpointPort{ @@ -177,7 +171,7 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context } log.V(1).Info("HCP kubernetes endpointslice reconciled", - "address", addr.IP, "hostname", addr.Hostname, "port", port) + "address", addr.IP, "host", host, "port", port) return nil } @@ -248,7 +242,7 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cl return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err) } - log.V(1).Info("HCP kubernetes endpoints reconciled", "address", addr.IP, "hostname", addr.Hostname, "port", port) + log.V(1).Info("HCP kubernetes endpoints reconciled", "address", addr.IP, "host", host, "port", port) return nil } @@ -288,8 +282,9 @@ func parseHCPHostPort(rawURL string) (string, int32, error) { // hcpEndpointAddress builds a corev1.EndpointAddress from the externally // reachable host. Endpoints require an IP; if the host is a DNS name we -// resolve it and keep the original name as Hostname so logs/events remain -// human-readable. +// resolve it. The Hostname field is intentionally left unset: the +// kubernetes API validates it as a DNS-1123 label (no dots), so an FQDN +// like "host.example.com" would be rejected. func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { if ip := net.ParseIP(host); ip != nil { if ip.IsLoopback() { @@ -317,8 +312,8 @@ func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { } if v4 := filteredIPs[0].To4(); v4 != nil { - return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil + return corev1.EndpointAddress{IP: v4.String()}, nil } - return corev1.EndpointAddress{IP: filteredIPs[0].String(), Hostname: host}, nil + return corev1.EndpointAddress{IP: filteredIPs[0].String()}, nil } diff --git a/tests/e2e/cluster_create_test.go b/tests/e2e/cluster_create_test.go index 9e2d2d5a..700096a3 100644 --- a/tests/e2e/cluster_create_test.go +++ b/tests/e2e/cluster_create_test.go @@ -5,8 +5,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + k3kcluster "github.com/rancher/k3k/pkg/controller/cluster" fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" @@ -61,3 +64,79 @@ var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTe Should(Succeed()) }) }) + +var _ = When("creating an HCP mode cluster", Label(e2eTestLabel), Label(slowTestsLabel), func() { + var virtualCluster *VirtualCluster + + BeforeEach(func() { + namespace := fwk3k.CreateNamespace(k8s) + + DeferCleanup(func() { + fwk3k.DeleteNamespaces(k8s, namespace.Name) + }) + + cluster := NewCluster(namespace.Name) + cluster.Spec.Mode = v1beta1.HCPClusterMode + + CreateCluster(cluster) + client, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: client, + } + }) + + It("is up and running", func() { + Eventually(func(g Gomega) { + ctx := GinkgoT().Context() + key := client.ObjectKeyFromObject(virtualCluster.Cluster) + g.Expect(k8sClient.Get(ctx, key, virtualCluster.Cluster)).To(Succeed()) + g.Expect(virtualCluster.Cluster.Status.Phase).To(BeEquivalentTo(v1beta1.ClusterReady)) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + }) + + It("creates a populated token secret", func() { + ctx := GinkgoT().Context() + + var tokenSecret corev1.Secret + err := k8sClient.Get(ctx, client.ObjectKey{ + Name: k3kcluster.TokenSecretName(virtualCluster.Cluster.Name), + Namespace: virtualCluster.Cluster.Namespace, + }, &tokenSecret) + Expect(err).NotTo(HaveOccurred()) + Expect(tokenSecret.Data["token"]).NotTo(BeEmpty()) + }) + + It("reconciles default/kubernetes Endpoints and EndpointSlice", func() { + ctx := GinkgoT().Context() + + Eventually(func(g Gomega) { + endpoints, err := virtualCluster.Client.CoreV1().Endpoints("default").Get(ctx, "kubernetes", metav1.GetOptions{}) + g.Expect(err).To(Not(HaveOccurred())) + g.Expect(endpoints.Subsets).To(HaveLen(1)) + g.Expect(endpoints.Subsets[0].Addresses).To(HaveLen(1)) + g.Expect(endpoints.Subsets[0].Addresses[0].IP).To(Equal(hostIP)) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + + Eventually(func(g Gomega) { + slices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").List(ctx, metav1.ListOptions{ + LabelSelector: "kubernetes.io/service-name=kubernetes", + }) + g.Expect(err).To(Not(HaveOccurred())) + g.Expect(slices.Items).To(HaveLen(1)) + g.Expect(slices.Items[0].Endpoints).To(HaveLen(1)) + g.Expect(slices.Items[0].Endpoints[0].Addresses).To(ContainElement(hostIP)) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + }) +}) From 9d0feeef0da3bf6439acb8f9d6f2af1ea5cd0ef2 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 15 Jun 2026 16:10:07 +0200 Subject: [PATCH 28/36] fix lint --- tests/e2e/cluster_create_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/cluster_create_test.go b/tests/e2e/cluster_create_test.go index 700096a3..47735d04 100644 --- a/tests/e2e/cluster_create_test.go +++ b/tests/e2e/cluster_create_test.go @@ -104,10 +104,12 @@ var _ = When("creating an HCP mode cluster", Label(e2eTestLabel), Label(slowTest ctx := GinkgoT().Context() var tokenSecret corev1.Secret + err := k8sClient.Get(ctx, client.ObjectKey{ Name: k3kcluster.TokenSecretName(virtualCluster.Cluster.Name), Namespace: virtualCluster.Cluster.Namespace, }, &tokenSecret) + Expect(err).NotTo(HaveOccurred()) Expect(tokenSecret.Data["token"]).NotTo(BeEmpty()) }) From 5ed97d942e14722f54ad58be59663f2f89314502 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 15 Jun 2026 23:01:32 +0200 Subject: [PATCH 29/36] fix tests --- tests/e2e/cluster_create_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/e2e/cluster_create_test.go b/tests/e2e/cluster_create_test.go index 47735d04..3746a345 100644 --- a/tests/e2e/cluster_create_test.go +++ b/tests/e2e/cluster_create_test.go @@ -118,24 +118,31 @@ var _ = When("creating an HCP mode cluster", Label(e2eTestLabel), Label(slowTest ctx := GinkgoT().Context() Eventually(func(g Gomega) { + key := client.ObjectKeyFromObject(virtualCluster.Cluster) + g.Expect(k8sClient.Get(ctx, key, virtualCluster.Cluster)).To(Succeed()) + endpoints, err := virtualCluster.Client.CoreV1().Endpoints("default").Get(ctx, "kubernetes", metav1.GetOptions{}) g.Expect(err).To(Not(HaveOccurred())) g.Expect(endpoints.Subsets).To(HaveLen(1)) g.Expect(endpoints.Subsets[0].Addresses).To(HaveLen(1)) - g.Expect(endpoints.Subsets[0].Addresses[0].IP).To(Equal(hostIP)) + g.Expect(virtualCluster.Cluster.Status.TLSSANs).To(ContainElement(endpoints.Subsets[0].Addresses[0].IP)) }). WithTimeout(time.Minute). WithPolling(time.Second). Should(Succeed()) Eventually(func(g Gomega) { + key := client.ObjectKeyFromObject(virtualCluster.Cluster) + g.Expect(k8sClient.Get(ctx, key, virtualCluster.Cluster)).To(Succeed()) + slices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").List(ctx, metav1.ListOptions{ LabelSelector: "kubernetes.io/service-name=kubernetes", }) g.Expect(err).To(Not(HaveOccurred())) g.Expect(slices.Items).To(HaveLen(1)) g.Expect(slices.Items[0].Endpoints).To(HaveLen(1)) - g.Expect(slices.Items[0].Endpoints[0].Addresses).To(ContainElement(hostIP)) + g.Expect(slices.Items[0].Endpoints[0].Addresses).To(HaveLen(1)) + g.Expect(virtualCluster.Cluster.Status.TLSSANs).To(ContainElement(slices.Items[0].Endpoints[0].Addresses[0])) }). WithTimeout(time.Minute). WithPolling(time.Second). From 65c30fb7554d1e6d0e98012dc956baaba426aca2 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Tue, 16 Jun 2026 12:16:48 +0200 Subject: [PATCH 30/36] addressed comments --- cli/cmds/kubeconfig.go | 3 +- pkg/controller/cluster/cluster.go | 19 +++-- pkg/controller/cluster/hcp.go | 62 +++++++------- pkg/controller/cluster/hcp_test.go | 35 ++++++-- pkg/controller/cluster/server/config.go | 2 +- pkg/controller/cluster/server/endpoint.go | 18 +++-- .../cluster/server/endpoint_test.go | 80 +++++++++++++++++++ pkg/controller/cluster/status.go | 12 +++ 8 files changed, 179 insertions(+), 52 deletions(-) create mode 100644 pkg/controller/cluster/server/endpoint_test.go diff --git a/cli/cmds/kubeconfig.go b/cli/cmds/kubeconfig.go index 1b3ad1b5..8d6e3cac 100644 --- a/cli/cmds/kubeconfig.go +++ b/cli/cmds/kubeconfig.go @@ -5,7 +5,6 @@ import ( "net/url" "os" "path/filepath" - "strings" "time" "github.com/sirupsen/logrus" @@ -147,7 +146,7 @@ func resolveServerHost(restConfigHost, override string) (string, error) { return "", err } - return strings.Split(u.Host, ":")[0], nil + return u.Hostname(), nil } func writeKubeconfigFile(cluster *v1beta1.Cluster, kubeconfig *clientcmdapi.Config, configName string) error { diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index 9a4601fd..6b5326bc 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -448,11 +448,13 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus return err } - // In hcp mode, derive the K3s installer command end-users run on their - // external nodes and surface it on the Cluster status. We also own the - // default/kubernetes Endpoints inside the virtual cluster (the apiserver - // reconciler is disabled for HCP) so external-node pods can reach the - // in-cluster apiserver ClusterIP. + // In hcp mode, validate that the cluster has an externally-routable + // API server endpoint (NodePort / LoadBalancer / Ingress) so external + // workers can join. The join command itself is printed by the CLI + // (`k3kcli cluster create` / `k3kcli kubeconfig generate`). + // We also own the default/kubernetes Endpoints/EndpointSlice inside the + // virtual cluster (the apiserver reconciler is disabled for HCP) so + // external-node pods can reach the in-cluster apiserver ClusterIP. if cluster.Spec.Mode == v1beta1.HCPClusterMode { if err := c.ensureHCPRegistration(ctx, cluster); err != nil { return err @@ -916,9 +918,10 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet } func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error { - // hcp mode is BYO-node by design: external (out-of-host-cluster) nodes join - // using the standard K3s installer command surfaced via Status.HCPRegistration. - // k3k therefore does not provision any agent pods on the host cluster. + // hcp mode is BYO-node by design: external (out-of-host-cluster) nodes + // join using the standard K3s installer command, which is printed by + // `k3kcli cluster create` / `k3kcli kubeconfig generate`. k3k therefore + // does not provision any agent pods on the host cluster. if cluster.Spec.Mode == v1beta1.HCPClusterMode { return nil } diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go index c6d5f299..18d44fd0 100644 --- a/pkg/controller/cluster/hcp.go +++ b/pkg/controller/cluster/hcp.go @@ -2,12 +2,12 @@ package cluster import ( "context" + "errors" "fmt" "net" "net/url" "strconv" - "k8s.io/apimachinery/pkg/api/meta" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" corev1 "k8s.io/api/core/v1" @@ -19,13 +19,22 @@ import ( "github.com/rancher/k3k/pkg/controller/cluster/server" ) -// ensureHCPRegistration computes the K3s installer command external nodes can -// run to join an HCP-mode cluster and stores it on cluster.Status.HCPRegistration. +// ErrHCPNoExternalEndpoint is returned by ensureHCPRegistration when an +// HCP-mode cluster has no externally-routable endpoint (no NodePort, +// LoadBalancer or Ingress) so external worker nodes cannot reach the API +// server. updateStatus translates it into a Ready=False condition with +// reason HCPNoExternalEndpoint instead of failing the reconcile outright. +var ErrHCPNoExternalEndpoint = errors.New("HCP cluster has no external endpoint") + +// ensureHCPRegistration verifies that an HCP-mode cluster exposes an +// externally-routable API server endpoint so external worker nodes can join. +// Join instructions (the `curl ... | sh -` line) are printed by the CLI +// (`k3kcli cluster create` / `k3kcli kubeconfig generate`); the controller +// does not persist them on the Cluster object. // -// When the cluster's Service is not externally reachable (no NodePort, -// LoadBalancer or Ingress configured) the command cannot be built; in that -// case the Ready condition is set to False with reason HCPNoExternalEndpoint -// so the operator surfaces the problem without failing the reconciliation. +// Returns ErrHCPNoExternalEndpoint when no NodePort, LoadBalancer or Ingress +// is configured, which updateStatus surfaces as Ready=False with reason +// HCPNoExternalEndpoint. func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster) error { log := ctrl.LoggerFrom(ctx) @@ -35,15 +44,10 @@ func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster * } if !external { - log.Info("HCP cluster has no externally-routable endpoint; skipping registration command", + log.Info("HCP cluster has no externally-routable endpoint", "cluster", cluster.Name, "namespace", cluster.Namespace) - meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ - Type: ConditionReady, - Status: metav1.ConditionFalse, - Reason: ReasonHCPNoExternalEndpoint, - Message: "HCP cluster has no external endpoint; set spec.expose.nodePort, spec.expose.loadBalancer or spec.expose.ingress so external nodes can reach the API server", - }) + return ErrHCPNoExternalEndpoint } return nil @@ -103,8 +107,9 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context } if !external { - // ensureHCPRegistration already surfaces this via Ready=False; - // nothing for us to do here. + // Defensive: reconcile would have already short-circuited with + // ErrHCPNoExternalEndpoint via ensureHCPRegistration before reaching + // here, but skip gracefully if invoked directly. return nil } @@ -113,7 +118,7 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err) } - addr, err := hcpEndpointAddress(host) + addr, err := hcpEndpointAddress(ctx, host) if err != nil { return err } @@ -185,8 +190,9 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cl } if !external { - // ensureHCPRegistration already surfaces this via Ready=False; - // nothing for us to do here. + // Defensive: reconcile would have already short-circuited with + // ErrHCPNoExternalEndpoint via ensureHCPRegistration before reaching + // here, but skip gracefully if invoked directly. return nil } @@ -195,7 +201,7 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cl return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err) } - addr, err := hcpEndpointAddress(host) + addr, err := hcpEndpointAddress(ctx, host) if err != nil { return err } @@ -282,10 +288,10 @@ func parseHCPHostPort(rawURL string) (string, int32, error) { // hcpEndpointAddress builds a corev1.EndpointAddress from the externally // reachable host. Endpoints require an IP; if the host is a DNS name we -// resolve it. The Hostname field is intentionally left unset: the -// kubernetes API validates it as a DNS-1123 label (no dots), so an FQDN -// like "host.example.com" would be rejected. -func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { +// resolve it. The Hostname field is intentionally left unset: +// the kubernetes API validates it as a DNS-1123 label (no dots), +// so an FQDN like "host.example.com" would be rejected. +func hcpEndpointAddress(ctx context.Context, host string) (corev1.EndpointAddress, error) { if ip := net.ParseIP(host); ip != nil { if ip.IsLoopback() { return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is a loopback address and cannot be used", host) @@ -294,16 +300,16 @@ func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { return corev1.EndpointAddress{IP: host}, nil } - ips, err := net.LookupIP(host) + ipAddrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) if err != nil { return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err) } var filteredIPs []net.IP - for _, ip := range ips { - if !ip.IsLoopback() { - filteredIPs = append(filteredIPs, ip) + for _, addr := range ipAddrs { + if !addr.IP.IsLoopback() { + filteredIPs = append(filteredIPs, addr.IP) } } diff --git a/pkg/controller/cluster/hcp_test.go b/pkg/controller/cluster/hcp_test.go index c85ef986..f264d2b2 100644 --- a/pkg/controller/cluster/hcp_test.go +++ b/pkg/controller/cluster/hcp_test.go @@ -2,11 +2,11 @@ package cluster import ( "context" + "errors" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -35,7 +35,7 @@ func Test_ensureHCPRegistration(t *testing.T) { }, } - t.Run("clusterip-only service sets degraded condition and clears registration", func(t *testing.T) { + t.Run("clusterip-only service returns ErrHCPNoExternalEndpoint", func(t *testing.T) { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: server.ServiceName(cluster.Name), @@ -54,12 +54,31 @@ func Test_ensureHCPRegistration(t *testing.T) { r := &ClusterReconciler{Client: fakeClient} c := cluster.DeepCopy() - require.NoError(t, r.ensureHCPRegistration(context.Background(), c)) + err := r.ensureHCPRegistration(context.Background(), c) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrHCPNoExternalEndpoint)) + }) - cond := meta.FindStatusCondition(c.Status.Conditions, ConditionReady) - require.NotNil(t, cond) - assert.Equal(t, metav1.ConditionFalse, cond.Status) - assert.Equal(t, ReasonHCPNoExternalEndpoint, cond.Reason) + t.Run("nodeport service returns no error", func(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: server.ServiceName(cluster.Name), + Namespace: cluster.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + ClusterIP: "10.43.0.50", + Ports: []corev1.ServicePort{ + {Name: "k3s-server-port", Port: 443, NodePort: 30443}, + }, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build() + r := &ClusterReconciler{Client: fakeClient} + + c := cluster.DeepCopy() + assert.NoError(t, r.ensureHCPRegistration(context.Background(), c)) }) } @@ -240,7 +259,7 @@ func Test_hcpEndpointAddress(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := hcpEndpointAddress(tt.input) + got, err := hcpEndpointAddress(context.Background(), tt.input) if tt.wantErr { require.Error(t, err) return diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index c24279d1..f4635003 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -79,7 +79,7 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok // shared and hcp modes both run K3s with --disable-agent (agentless server). switch cluster.Spec.Mode { - case v1beta1.SharedClusterMode: + case "", v1beta1.SharedClusterMode: serverConfig.DisableAgent = true serverConfig.EgressSelectorMode = "disabled" serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"} diff --git a/pkg/controller/cluster/server/endpoint.go b/pkg/controller/cluster/server/endpoint.go index 23962630..02736f17 100644 --- a/pkg/controller/cluster/server/endpoint.go +++ b/pkg/controller/cluster/server/endpoint.go @@ -51,12 +51,20 @@ func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, h port = k3kService.Spec.Ports[0].NodePort } case corev1.ServiceTypeLoadBalancer: - external = true - if len(k3kService.Status.LoadBalancer.Ingress) > 0 { - ip = k3kService.Status.LoadBalancer.Ingress[0].IP - } else { - logrus.Warn("No ingress found in LoadBalancer service.") + ingress := k3kService.Status.LoadBalancer.Ingress[0] + switch { + case ingress.IP != "": + ip = ingress.IP + external = true + case ingress.Hostname != "": + ip = ingress.Hostname + external = true + } + } + + if !external { + logrus.Warn("No usable ingress address found in LoadBalancer service.") } if len(k3kService.Spec.Ports) > 0 { diff --git a/pkg/controller/cluster/server/endpoint_test.go b/pkg/controller/cluster/server/endpoint_test.go new file mode 100644 index 00000000..22075c14 --- /dev/null +++ b/pkg/controller/cluster/server/endpoint_test.go @@ -0,0 +1,80 @@ +package server + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" +) + +func Test_ServerURL_LoadBalancer(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + + cluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "team-a"}, + Status: v1beta1.ClusterStatus{TLSSANs: []string{"203.0.113.10", "lb.example.com"}}, + } + + tests := []struct { + name string + ingress []corev1.LoadBalancerIngress + wantURL string + wantExternal bool + }{ + { + name: "no ingress yet (LB still provisioning)", + ingress: nil, + wantExternal: false, + }, + { + name: "ingress with IP", + ingress: []corev1.LoadBalancerIngress{{IP: "203.0.113.10"}}, + wantURL: "https://203.0.113.10", + wantExternal: true, + }, + { + name: "ingress with hostname only", + ingress: []corev1.LoadBalancerIngress{{Hostname: "lb.example.com"}}, + wantURL: "https://lb.example.com", + wantExternal: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: ServiceName(cluster.Name), + Namespace: cluster.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + ClusterIP: "10.43.0.50", + Ports: []corev1.ServicePort{{Name: "k3s-server-port", Port: 443}}, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{Ingress: tt.ingress}, + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build() + url, external, err := ServerURL(context.Background(), c, cluster, "", 0) + require.NoError(t, err) + assert.Equal(t, tt.wantExternal, external) + + if tt.wantExternal { + assert.Equal(t, tt.wantURL, url) + } + }) + } +} diff --git a/pkg/controller/cluster/status.go b/pkg/controller/cluster/status.go index cff1024d..7c61ed3f 100644 --- a/pkg/controller/cluster/status.go +++ b/pkg/controller/cluster/status.go @@ -70,6 +70,18 @@ func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.C return } + if errors.Is(reconcileErr, ErrHCPNoExternalEndpoint) { + cluster.Status.Phase = v1beta1.ClusterPending + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: ConditionReady, + Status: metav1.ConditionFalse, + Reason: ReasonHCPNoExternalEndpoint, + Message: "HCP cluster has no external endpoint; set spec.expose.nodePort, spec.expose.loadBalancer or spec.expose.ingress so external nodes can reach the API server", + }) + + return + } + // If there's an error, but it's not a validation error, the cluster is in a failed state. if reconcileErr != nil { cluster.Status.Phase = v1beta1.ClusterFailed From 78b433eff0c2afb0fffdb11ef9ae65c6f9a79dab Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Tue, 16 Jun 2026 16:15:17 +0200 Subject: [PATCH 31/36] removed enum definition from ClusterMode type --- charts/k3k/templates/crds/k3k.io_clusters.yaml | 13 ++++--------- .../crds/k3k.io_virtualclusterpolicies.yaml | 13 ++++--------- docs/crds/crds.adoc | 3 +-- docs/crds/crds.md | 3 +-- pkg/apis/k3k.io/v1beta1/types.go | 3 --- 5 files changed, 10 insertions(+), 25 deletions(-) diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index 80d62d19..84af9d78 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -1289,19 +1289,14 @@ spec: are mirrored into the virtual cluster. type: boolean mode: - allOf: - - enum: - - shared - - virtual - - hcp - - enum: - - shared - - virtual - - hcp default: shared description: |- Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp". Defaults to "shared". This field is immutable. + enum: + - shared + - virtual + - hcp type: string x-kubernetes-validations: - message: mode is immutable diff --git a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml index a708cca4..e4996125 100644 --- a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml +++ b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml @@ -50,17 +50,12 @@ spec: description: Spec defines the desired state of the VirtualClusterPolicy. properties: allowedMode: - allOf: - - enum: - - shared - - virtual - - hcp - - enum: - - shared - - virtual - - hcp default: shared description: AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". + enum: + - shared + - virtual + - hcp type: string x-kubernetes-validations: - message: mode is immutable diff --git a/docs/crds/crds.adoc b/docs/crds/crds.adoc index cd57d5d8..152630a2 100644 --- a/docs/crds/crds.adoc +++ b/docs/crds/crds.adoc @@ -133,8 +133,7 @@ _Underlying type:_ _string_ ClusterMode is the possible provisioning mode of a Cluster. -_Validation:_ -- Enum: [shared virtual hcp] + _Appears In:_ diff --git a/docs/crds/crds.md b/docs/crds/crds.md index a43bd48b..406286c8 100644 --- a/docs/crds/crds.md +++ b/docs/crds/crds.md @@ -102,8 +102,7 @@ _Underlying type:_ _string_ ClusterMode is the possible provisioning mode of a Cluster. -_Validation:_ -- Enum: [shared virtual hcp] + _Appears in:_ - [ClusterSpec](#clusterspec) diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index 59f47198..fcf5c81f 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -412,9 +412,6 @@ type StorageClassSyncConfig struct { } // ClusterMode is the possible provisioning mode of a Cluster. -// -// +kubebuilder:validation:Enum=shared;virtual;hcp -// +kubebuilder:default="shared" type ClusterMode string const ( From 27676754eea32f817c7719b4c3db355639caff37 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 17 Jun 2026 12:03:29 +0200 Subject: [PATCH 32/36] added enum docs --- docs/crds/crds.adoc | 2 ++ docs/crds/crds.md | 2 ++ pkg/apis/k3k.io/v1beta1/types.go | 2 ++ 3 files changed, 6 insertions(+) diff --git a/docs/crds/crds.adoc b/docs/crds/crds.adoc index 152630a2..43c2d5ac 100644 --- a/docs/crds/crds.adoc +++ b/docs/crds/crds.adoc @@ -133,6 +133,8 @@ _Underlying type:_ _string_ ClusterMode is the possible provisioning mode of a Cluster. +Supported values: `shared`, `virtual`, `hcp`. + _Appears In:_ diff --git a/docs/crds/crds.md b/docs/crds/crds.md index 406286c8..08342601 100644 --- a/docs/crds/crds.md +++ b/docs/crds/crds.md @@ -102,6 +102,8 @@ _Underlying type:_ _string_ ClusterMode is the possible provisioning mode of a Cluster. +Supported values: `shared`, `virtual`, `hcp`. + _Appears in:_ diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index fcf5c81f..f21ad87a 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -412,6 +412,8 @@ type StorageClassSyncConfig struct { } // ClusterMode is the possible provisioning mode of a Cluster. +// +// Supported values: `shared`, `virtual`, `hcp`. type ClusterMode string const ( From 23058e0a319b55f9352c333e05ecb5633d7bbe63 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 17 Jun 2026 12:09:29 +0200 Subject: [PATCH 33/36] align docs --- charts/k3k/templates/crds/k3k.io_clusters.yaml | 2 +- docs/crds/crds.adoc | 7 ++++--- docs/crds/crds.md | 7 ++++--- pkg/apis/k3k.io/v1beta1/types.go | 11 ++++++----- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index 84af9d78..03aa27ef 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -1416,7 +1416,7 @@ spec: role: description: |- Role is the type of the k3k pod that will be used to mount the secret. - This can be 'server', 'agent', or 'all' (for both). + This can be `server`, `agent`, or `all` (for both). enum: - server - agent diff --git a/docs/crds/crds.adoc b/docs/crds/crds.adoc index 43c2d5ac..3c75619b 100644 --- a/docs/crds/crds.adoc +++ b/docs/crds/crds.adoc @@ -548,8 +548,9 @@ _Underlying type:_ _string_ PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace. -_Validation:_ -- Enum: [privileged baseline restricted] +Supported values: `privileged`, `baseline`, `restricted`. + + _Appears In:_ @@ -619,7 +620,7 @@ secret contents will be mounted. + | | When specified, only the specified key from the secret will be mounted as a file + at MountPath, keeping the parent directory writable. + | | | *`role`* __string__ | Role is the type of the k3k pod that will be used to mount the secret. + -This can be 'server', 'agent', or 'all' (for both). + | | Enum: [server agent all] + +This can be `server`, `agent`, or `all` (for both). + | | Enum: [server agent all] + |=== diff --git a/docs/crds/crds.md b/docs/crds/crds.md index 08342601..ebeb197f 100644 --- a/docs/crds/crds.md +++ b/docs/crds/crds.md @@ -410,8 +410,9 @@ _Underlying type:_ _string_ PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace. -_Validation:_ -- Enum: [privileged baseline restricted] +Supported values: `privileged`, `baseline`, `restricted`. + + _Appears in:_ - [VirtualClusterPolicySpec](#virtualclusterpolicyspec) @@ -455,7 +456,7 @@ _Appears in:_ | `optional` _boolean_ | optional field specify whether the Secret or its keys must be defined | | | | `mountPath` _string_ | MountPath is the path within server and agent pods where the
secret contents will be mounted. | | | | `subPath` _string_ | SubPath is an optional path within the secret to mount instead of the root.
When specified, only the specified key from the secret will be mounted as a file
at MountPath, keeping the parent directory writable. | | | -| `role` _string_ | Role is the type of the k3k pod that will be used to mount the secret.
This can be 'server', 'agent', or 'all' (for both). | | Enum: [server agent all]
| +| `role` _string_ | Role is the type of the k3k pod that will be used to mount the secret.
This can be `server`, `agent`, or `all` (for both). | | Enum: [server agent all]
| #### SecretSyncConfig diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index f21ad87a..199b1471 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -48,7 +48,7 @@ type ClusterSpec struct { // Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp". // Defaults to "shared". This field is immutable. // - // +kubebuilder:default="shared" + // +kubebuilder:default=shared // +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf" // +optional @@ -253,10 +253,10 @@ type SecretMount struct { // +optional SubPath string `json:"subPath,omitempty"` // Role is the type of the k3k pod that will be used to mount the secret. - // This can be 'server', 'agent', or 'all' (for both). + // This can be `server`, `agent`, or `all` (for both). // - // +optional // +kubebuilder:validation:Enum=server;agent;all + // +optional Role string `json:"role,omitempty"` } @@ -630,7 +630,7 @@ type ClusterStatus struct { // Phase is a high-level summary of the cluster's current lifecycle state. // - // +kubebuilder:default="Unknown" + // +kubebuilder:default=Unknown // +kubebuilder:validation:Enum=Pending;Provisioning;Ready;Failed;Terminating;Unknown // +optional Phase ClusterPhase `json:"phase,omitempty"` @@ -790,6 +790,7 @@ type VirtualClusterPolicySpec struct { // PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. // + // +kubebuilder:validation:Enum=privileged;baseline;restricted // +optional PodSecurityAdmissionLevel *PodSecurityAdmissionLevel `json:"podSecurityAdmissionLevel,omitempty"` @@ -822,7 +823,7 @@ type VirtualClusterPolicySpec struct { // PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace. // -// +kubebuilder:validation:Enum=privileged;baseline;restricted +// Supported values: `privileged`, `baseline`, `restricted`. type PodSecurityAdmissionLevel string const ( From 4debe449116852ff266f9c884457c35bebe408af Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 17 Jun 2026 12:13:01 +0200 Subject: [PATCH 34/36] fix possible flaky test --- tests/e2e/cluster_create_test.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/e2e/cluster_create_test.go b/tests/e2e/cluster_create_test.go index 3746a345..6856a7fd 100644 --- a/tests/e2e/cluster_create_test.go +++ b/tests/e2e/cluster_create_test.go @@ -103,15 +103,20 @@ var _ = When("creating an HCP mode cluster", Label(e2eTestLabel), Label(slowTest It("creates a populated token secret", func() { ctx := GinkgoT().Context() - var tokenSecret corev1.Secret + Eventually(func(g Gomega) { + var tokenSecret corev1.Secret - err := k8sClient.Get(ctx, client.ObjectKey{ - Name: k3kcluster.TokenSecretName(virtualCluster.Cluster.Name), - Namespace: virtualCluster.Cluster.Namespace, - }, &tokenSecret) + key := client.ObjectKey{ + Name: k3kcluster.TokenSecretName(virtualCluster.Cluster.Name), + Namespace: virtualCluster.Cluster.Namespace, + } - Expect(err).NotTo(HaveOccurred()) - Expect(tokenSecret.Data["token"]).NotTo(BeEmpty()) + g.Expect(k8sClient.Get(ctx, key, &tokenSecret)).To(Succeed()) + g.Expect(tokenSecret.Data["token"]).NotTo(BeEmpty()) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) }) It("reconciles default/kubernetes Endpoints and EndpointSlice", func() { From f8063cec49faab3891572997fb0d4b15af4eba57 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 17 Jun 2026 12:26:32 +0200 Subject: [PATCH 35/36] ipv6 fix --- pkg/controller/cluster/server/endpoint.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/controller/cluster/server/endpoint.go b/pkg/controller/cluster/server/endpoint.go index 02736f17..66282a5e 100644 --- a/pkg/controller/cluster/server/endpoint.go +++ b/pkg/controller/cluster/server/endpoint.go @@ -3,7 +3,9 @@ package server import ( "context" "fmt" + "net" "slices" + "strconv" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/types" @@ -92,10 +94,7 @@ func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, h } } - url := "https://" + ip - if port != httpsPort { - url = fmt.Sprintf("%s:%d", url, port) - } + url := "https://" + net.JoinHostPort(ip, strconv.Itoa(int(port))) // if ingress is specified, use the ingress host if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { From 367f1606a08e3f00a74666aa9a3fa83a0852d92d Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 17 Jun 2026 12:39:41 +0200 Subject: [PATCH 36/36] fix test --- pkg/controller/cluster/server/endpoint.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/controller/cluster/server/endpoint.go b/pkg/controller/cluster/server/endpoint.go index 66282a5e..29ad8e77 100644 --- a/pkg/controller/cluster/server/endpoint.go +++ b/pkg/controller/cluster/server/endpoint.go @@ -94,7 +94,12 @@ func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, h } } - url := "https://" + net.JoinHostPort(ip, strconv.Itoa(int(port))) + host := ip + if port != httpsPort { + host = net.JoinHostPort(ip, strconv.Itoa(int(port))) + } + + url := "https://" + host // if ingress is specified, use the ingress host if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {