From 87242ff00587d3073704f60bb1b11165a0b7a0c0 Mon Sep 17 00:00:00 2001 From: Parth Yadav Date: Sun, 11 Jan 2026 16:01:24 +0530 Subject: [PATCH] feat: extend Gateway API support to Konnectivity addons (#1054) This change extends Gateway API support to Konnectivity addons. When `spec.controlPlane.gateway` is configured and Konnectivity addon is enabled, Kamaji automatically creates two TLSRoutes: 1. A Control plane TLSRoute (port 6443, sectionName "kube-apiserver") 2. A Konnectivity TLSRoute (port 8132, sectionName "konnectivity-server") Both routes use the hostname specified in `gateway.hostname` and reference the same Gateway resource via `parentRefs`, with `port` and `sectionName` set automatically by Kamaji. This patch also adds CEL validation to prevent users from specifying `port` or `sectionName` in Gateway `parentRefs`, as these fields are now managed automatically by Kamaji. Signed-off-by: Parth Yadav --- Makefile | 6 +- api/v1alpha1/tenantcontrolplane_status.go | 1 + api/v1alpha1/tenantcontrolplane_types.go | 1 + api/v1alpha1/zz_generated.deepcopy.go | 5 + ...i.clastix.io_tenantcontrolplanes_spec.yaml | 380 +++++++++++++ ...kamaji.clastix.io_tenantcontrolplanes.yaml | 380 +++++++++++++ ...i_v1alpha1_tenantcontrolplane_gateway.yaml | 81 +++ controllers/resources.go | 9 + docs/content/guides/gateway-api.md | 213 ++++++++ docs/content/reference/api.md | 506 ++++++++++++++++++ docs/mkdocs.yml | 1 + e2e/suite_test.go | 32 ++ e2e/tcp_gateway_konnectivity_ready_test.go | 151 ++++++ e2e/tcp_gateway_ready_test.go | 32 +- e2e/utils_test.go | 59 ++ internal/resources/k8s_gateway_resource.go | 212 +------- .../resources/k8s_gateway_resource_test.go | 134 +++++ internal/resources/k8s_gateway_utils.go | 241 +++++++++ internal/resources/konnectivity/agent.go | 15 + .../konnectivity/gateway_resource.go | 232 ++++++++ .../konnectivity/gateway_resource_test.go | 218 ++++++++ internal/resources/konnectivity/metrics.go | 1 + 22 files changed, 2711 insertions(+), 199 deletions(-) create mode 100644 config/samples/kamaji_v1alpha1_tenantcontrolplane_gateway.yaml create mode 100644 docs/content/guides/gateway-api.md create mode 100644 e2e/tcp_gateway_konnectivity_ready_test.go create mode 100644 internal/resources/k8s_gateway_utils.go create mode 100644 internal/resources/konnectivity/gateway_resource.go create mode 100644 internal/resources/konnectivity/gateway_resource_test.go diff --git a/Makefile b/Makefile index 6772e6f..d97a158 100644 --- a/Makefile +++ b/Makefile @@ -248,6 +248,10 @@ gateway-api: kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.0/experimental-install.yaml kubectl wait --for=condition=Established crd/gateways.gateway.networking.k8s.io --timeout=60s +envoy-gateway: gateway-api helm ## Install Envoy Gateway for Gateway API tests. + $(HELM) upgrade --install eg oci://docker.io/envoyproxy/gateway-helm --version v1.6.1 -n envoy-gateway-system --create-namespace + kubectl wait --timeout=5m -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available + load: kind $(KIND) load docker-image --name kamaji ${CONTAINER_REPOSITORY}:${VERSION} @@ -261,7 +265,7 @@ cleanup: kind $(KIND) delete cluster --name kamaji .PHONY: e2e -e2e: env build load helm ginkgo cert-manager gateway-api ## Create a KinD cluster, install Kamaji on it and run the test suite. +e2e: env build load helm ginkgo cert-manager gateway-api envoy-gateway ## Create a KinD cluster, install Kamaji on it and run the test suite. $(HELM) upgrade --debug --install kamaji-crds ./charts/kamaji-crds --create-namespace --namespace kamaji-system $(HELM) repo add clastix https://clastix.github.io/charts $(HELM) dependency build ./charts/kamaji diff --git a/api/v1alpha1/tenantcontrolplane_status.go b/api/v1alpha1/tenantcontrolplane_status.go index b5c5df7..85ce020 100644 --- a/api/v1alpha1/tenantcontrolplane_status.go +++ b/api/v1alpha1/tenantcontrolplane_status.go @@ -139,6 +139,7 @@ type KonnectivityStatus struct { ClusterRoleBinding ExternalKubernetesObjectStatus `json:"clusterrolebinding,omitempty"` Agent KonnectivityAgentStatus `json:"agent,omitempty"` Service KubernetesServiceStatus `json:"service,omitempty"` + Gateway *KubernetesGatewayStatus `json:"gateway,omitempty"` } type KonnectivityConfigMap struct { diff --git a/api/v1alpha1/tenantcontrolplane_types.go b/api/v1alpha1/tenantcontrolplane_types.go index a54c746..0b6c6d2 100644 --- a/api/v1alpha1/tenantcontrolplane_types.go +++ b/api/v1alpha1/tenantcontrolplane_types.go @@ -155,6 +155,7 @@ type IngressSpec struct { } // GatewaySpec defines the options for the Gateway which will expose API Server of the Tenant Control Plane. +// +kubebuilder:validation:XValidation:rule="!has(self.parentRefs) || size(self.parentRefs) == 0 || self.parentRefs.all(ref, !has(ref.port) && !has(ref.sectionName))",message="parentRefs must not specify port or sectionName, these are set automatically by Kamaji" type GatewaySpec struct { // AdditionalMetadata to add Labels and Annotations support. AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index def8423..c06daa0 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1058,6 +1058,11 @@ func (in *KonnectivityStatus) DeepCopyInto(out *KonnectivityStatus) { in.ClusterRoleBinding.DeepCopyInto(&out.ClusterRoleBinding) in.Agent.DeepCopyInto(&out.Agent) in.Service.DeepCopyInto(&out.Service) + if in.Gateway != nil { + in, out := &in.Gateway, &out.Gateway + *out = new(KubernetesGatewayStatus) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityStatus. diff --git a/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml b/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml index c54e094..bad7313 100644 --- a/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml +++ b/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml @@ -6896,6 +6896,9 @@ versions: type: object type: array type: object + x-kubernetes-validations: + - message: parentRefs must not specify port or sectionName, these are set automatically by Kamaji + rule: '!has(self.parentRefs) || size(self.parentRefs) == 0 || self.parentRefs.all(ref, !has(ref.port) && !has(ref.sectionName))' ingress: description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane properties: @@ -7349,6 +7352,383 @@ versions: type: object enabled: type: boolean + gateway: + description: KubernetesGatewayStatus defines the status for the Tenant Control Plane Gateway in the management cluster. + properties: + accessPoints: + description: A list of valid access points that the route exposes. + items: + properties: + port: + format: int32 + type: integer + type: + description: |- + AddressType defines how a network address is represented as a text string. + This may take two possible forms: + + * A predefined CamelCase string identifier (currently limited to `IPAddress` or `Hostname`) + * A domain-prefixed string identifier (like `acme.io/CustomAddressType`) + + Values `IPAddress` and `Hostname` have Extended support. + + The `NamedAddress` value has been deprecated in favor of implementation + specific domain-prefixed strings. + + All other values, including domain-prefixed values have Implementation-specific support, + which are used in implementation-specific behaviors. Support for additional + predefined CamelCase identifiers may be added in future releases. + maxLength: 253 + minLength: 1 + pattern: ^Hostname|IPAddress|NamedAddress|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + urls: + items: + type: string + type: array + value: + type: string + required: + - port + - type + - value + type: object + type: array + parents: + description: |- + Parents is a list of parent resources (usually Gateways) that are + associated with the route, and the status of the route with respect to + each parent. When this route attaches to a parent, the controller that + manages the parent must add an entry to this list when the controller + first sees the route and should update the entry as appropriate when the + route or gateway is modified. + + Note that parent references that cannot be resolved by an implementation + of this API will not be added to this list. Implementations of this API + can only populate Route status for the Gateways/parent resources they are + responsible for. + + A maximum of 32 Gateways will be represented in this list. An empty list + means the route has not been attached to any Gateway. + + + Notes for implementors: + + While parents is not a listType `map`, this is due to the fact that the + list key is not scalar, and Kubernetes is unable to represent this. + + Parent status MUST be considered to be namespaced by the combination of + the parentRef and controllerName fields, and implementations should keep + the following rules in mind when updating this status: + + * Implementations MUST update only entries that have a matching value of + `controllerName` for that implementation. + * Implementations MUST NOT update entries with non-matching `controllerName` + fields. + * Implementations MUST treat each `parentRef`` in the Route separately and + update its status based on the relationship with that parent. + * Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + + + items: + description: |- + RouteParentStatus describes the status of a route with respect to an + associated Parent. + properties: + conditions: + description: |- + Conditions describes the status of the route with respect to the Gateway. + Note that the route's availability is also subject to the Gateway's own + status conditions and listener status. + + If the Route's ParentRef specifies an existing Gateway that supports + Routes of this kind AND that Gateway's controller has sufficient access, + then that Gateway's controller MUST set the "Accepted" condition on the + Route, to indicate whether the route has been accepted or rejected by the + Gateway, and why. + + A Route MUST be considered "Accepted" if at least one of the Route's + rules is implemented by the Gateway. + + There are a number of cases where the "Accepted" condition may not be set + due to lack of controller visibility, that includes when: + + * The Route refers to a nonexistent parent. + * The Route is of a type that the controller does not support. + * The Route is in a namespace the controller does not have access to. + + + + Notes for implementors: + + Conditions are a listType `map`, which means that they function like a + map with a key of the `type` field _in the k8s apiserver_. + + This means that implementations must obey some rules when updating this + section. + + * Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + * Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. + * Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. + * Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. + * If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + + + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controllerName: + description: |- + ControllerName is a domain/path string that indicates the name of the + controller that wrote this status. This corresponds with the + controllerName field on GatewayClass. + + Example: "example.net/gateway-controller". + + The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are + valid Kubernetes names + (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + + Controllers MUST populate this field when writing status. Controllers should ensure that + entries to status populated with their ControllerName are cleaned up when they are no + longer necessary. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + parentRef: + description: |- + ParentRef corresponds with a ParentRef in the spec that this + RouteParentStatus struct describes the status of. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + required: + - conditions + - controllerName + - parentRef + type: object + maxItems: 32 + type: array + x-kubernetes-list-type: atomic + routeRef: + description: Reference to the route created for this tenant. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - parents + type: object kubeconfig: description: KubeconfigStatus contains information about the generated kubeconfig. properties: diff --git a/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml b/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml index aa0373f..a0d27c6 100644 --- a/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml +++ b/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml @@ -6904,6 +6904,9 @@ spec: type: object type: array type: object + x-kubernetes-validations: + - message: parentRefs must not specify port or sectionName, these are set automatically by Kamaji + rule: '!has(self.parentRefs) || size(self.parentRefs) == 0 || self.parentRefs.all(ref, !has(ref.port) && !has(ref.sectionName))' ingress: description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane properties: @@ -7357,6 +7360,383 @@ spec: type: object enabled: type: boolean + gateway: + description: KubernetesGatewayStatus defines the status for the Tenant Control Plane Gateway in the management cluster. + properties: + accessPoints: + description: A list of valid access points that the route exposes. + items: + properties: + port: + format: int32 + type: integer + type: + description: |- + AddressType defines how a network address is represented as a text string. + This may take two possible forms: + + * A predefined CamelCase string identifier (currently limited to `IPAddress` or `Hostname`) + * A domain-prefixed string identifier (like `acme.io/CustomAddressType`) + + Values `IPAddress` and `Hostname` have Extended support. + + The `NamedAddress` value has been deprecated in favor of implementation + specific domain-prefixed strings. + + All other values, including domain-prefixed values have Implementation-specific support, + which are used in implementation-specific behaviors. Support for additional + predefined CamelCase identifiers may be added in future releases. + maxLength: 253 + minLength: 1 + pattern: ^Hostname|IPAddress|NamedAddress|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + urls: + items: + type: string + type: array + value: + type: string + required: + - port + - type + - value + type: object + type: array + parents: + description: |- + Parents is a list of parent resources (usually Gateways) that are + associated with the route, and the status of the route with respect to + each parent. When this route attaches to a parent, the controller that + manages the parent must add an entry to this list when the controller + first sees the route and should update the entry as appropriate when the + route or gateway is modified. + + Note that parent references that cannot be resolved by an implementation + of this API will not be added to this list. Implementations of this API + can only populate Route status for the Gateways/parent resources they are + responsible for. + + A maximum of 32 Gateways will be represented in this list. An empty list + means the route has not been attached to any Gateway. + + + Notes for implementors: + + While parents is not a listType `map`, this is due to the fact that the + list key is not scalar, and Kubernetes is unable to represent this. + + Parent status MUST be considered to be namespaced by the combination of + the parentRef and controllerName fields, and implementations should keep + the following rules in mind when updating this status: + + * Implementations MUST update only entries that have a matching value of + `controllerName` for that implementation. + * Implementations MUST NOT update entries with non-matching `controllerName` + fields. + * Implementations MUST treat each `parentRef`` in the Route separately and + update its status based on the relationship with that parent. + * Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + + + items: + description: |- + RouteParentStatus describes the status of a route with respect to an + associated Parent. + properties: + conditions: + description: |- + Conditions describes the status of the route with respect to the Gateway. + Note that the route's availability is also subject to the Gateway's own + status conditions and listener status. + + If the Route's ParentRef specifies an existing Gateway that supports + Routes of this kind AND that Gateway's controller has sufficient access, + then that Gateway's controller MUST set the "Accepted" condition on the + Route, to indicate whether the route has been accepted or rejected by the + Gateway, and why. + + A Route MUST be considered "Accepted" if at least one of the Route's + rules is implemented by the Gateway. + + There are a number of cases where the "Accepted" condition may not be set + due to lack of controller visibility, that includes when: + + * The Route refers to a nonexistent parent. + * The Route is of a type that the controller does not support. + * The Route is in a namespace the controller does not have access to. + + + + Notes for implementors: + + Conditions are a listType `map`, which means that they function like a + map with a key of the `type` field _in the k8s apiserver_. + + This means that implementations must obey some rules when updating this + section. + + * Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + * Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. + * Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. + * Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. + * If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + + + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controllerName: + description: |- + ControllerName is a domain/path string that indicates the name of the + controller that wrote this status. This corresponds with the + controllerName field on GatewayClass. + + Example: "example.net/gateway-controller". + + The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are + valid Kubernetes names + (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + + Controllers MUST populate this field when writing status. Controllers should ensure that + entries to status populated with their ControllerName are cleaned up when they are no + longer necessary. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + parentRef: + description: |- + ParentRef corresponds with a ParentRef in the spec that this + RouteParentStatus struct describes the status of. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + required: + - conditions + - controllerName + - parentRef + type: object + maxItems: 32 + type: array + x-kubernetes-list-type: atomic + routeRef: + description: Reference to the route created for this tenant. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - parents + type: object kubeconfig: description: KubeconfigStatus contains information about the generated kubeconfig. properties: diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_gateway.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_gateway.yaml new file mode 100644 index 0000000..0eb3e06 --- /dev/null +++ b/config/samples/kamaji_v1alpha1_tenantcontrolplane_gateway.yaml @@ -0,0 +1,81 @@ +# Copyright 2022 Clastix Labs +# SPDX-License-Identifier: Apache-2.0 + +# This example demonstrates how to configure Gateway API support for a Tenant Control Plane. +# +# Prerequisites: +# 1. Gateway API CRDs must be installed (GatewayClass, Gateway, TLSRoute) +# 2. A Gateway resource must exist with listeners for ports 6443 and 8132 +# 3. DNS(or worker nodes hosts entries) must be configured to resolve the hostname to the Gateway's external address +# +# Example GatewayClass and Gateway configuration: +# +# apiVersion: gateway.networking.k8s.io/v1 +# kind: GatewayClass +# metadata: +# name: envoy-gw-class +# spec: +# controllerName: gateway.envoyproxy.io/gatewayclass-controller +# --- +# apiVersion: gateway.networking.k8s.io/v1 +# kind: Gateway +# metadata: +# name: gateway +# namespace: default +# spec: +# gatewayClassName: envoy-gw-class +# listeners: +# - allowedRoutes: +# kinds: +# - group: gateway.networking.k8s.io +# kind: TLSRoute +# namespaces: +# from: All +# hostname: '*.cluster.dev' +# name: kube-apiserver +# port: 6443 +# protocol: TLS +# tls: +# mode: Passthrough +# - allowedRoutes: +# kinds: +# - group: gateway.networking.k8s.io +# kind: TLSRoute +# namespaces: +# from: All +# hostname: '*.cluster.dev' +# name: konnectivity-server +# port: 8132 +# protocol: TLS +# tls: +# mode: Passthrough + +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: demo-tcp-1 +spec: + addons: + coreDNS: {} + kubeProxy: {} + konnectivity: {} + dataStore: default + controlPlane: + gateway: + hostname: "c11.cluster.dev" # worker nodes or kubectl clients must be able to resolve this hostname to the Gateway's external address. + parentRefs: + - name: gateway + namespace: default + deployment: + replicas: 1 + service: + serviceType: ClusterIP + kubernetes: + version: v1.32.0 + kubelet: + cgroupfs: systemd + networkProfile: + port: 6443 + certSANs: + - "c11.cluster.dev" + diff --git a/controllers/resources.go b/controllers/resources.go index f2c3737..acec173 100644 --- a/controllers/resources.go +++ b/controllers/resources.go @@ -74,6 +74,7 @@ func GetResources(ctx context.Context, config GroupResourceBuilderConfiguration) // Conditionally add Gateway resources if utilities.AreGatewayResourcesAvailable(ctx, config.client, config.DiscoveryClient) { resources = append(resources, getKubernetesGatewayResources(config.client)...) + resources = append(resources, getKonnectivityGatewayResources(config.client)...) } return resources @@ -146,6 +147,14 @@ func getKubernetesGatewayResources(c client.Client) []resources.Resource { } } +func getKonnectivityGatewayResources(c client.Client) []resources.Resource { + return []resources.Resource{ + &konnectivity.KubernetesKonnectivityGatewayResource{ + Client: c, + }, + } +} + func getKubeadmConfigResources(c client.Client, tmpDirectory string, dataStore kamajiv1alpha1.DataStore) []resources.Resource { var endpoints []string diff --git a/docs/content/guides/gateway-api.md b/docs/content/guides/gateway-api.md new file mode 100644 index 0000000..6c358e8 --- /dev/null +++ b/docs/content/guides/gateway-api.md @@ -0,0 +1,213 @@ +# Gateway API Support + +Kamaji provides built-in support for the [Gateway API](https://gateway-api.sigs.k8s.io/), allowing you to expose Tenant Control Planes using TLSRoute resources with SNI-based routing. This enables hostname-based routing to multiple Tenant Control Planes through a single Gateway resource, reducing the need for dedicated LoadBalancer services. + +## Overview + +Gateway API support in Kamaji automatically creates and manages TLSRoute resources for your Tenant Control Planes. When you configure a Gateway for a Tenant Control Plane, Kamaji automatically creates TLSRoutes for the Control Plane API Server. If konnectivity is enabled, a separate TLSRoute is created for it. Both TLSRoutes use the same hostname and Gateway resource, but route to different ports(listeners) using port-based routing and semantic `sectionName` values. + +Therefore, the target `Gateway` resource must have right listener configurations (see the Gateway [example section](#gateway-resource-setup) below). + + +## How It Works + +When you configure `spec.controlPlane.gateway` in a TenantControlPlane resource, Kamaji automatically: + +1. **Creates a TLSRoute for the control plane** that routes for port 6443 (or `spec.networkProfile.port`) with sectionName `"kube-apiserver"` +2. **Creates a TLSRoute for Konnectivity** (if konnectivity addon is enabled) that routes for port 8132 (or `spec.addons.konnectivity.server.port`) with sectionName `"konnectivity-server"` + +Both TLSRoutes: + +- Use the same hostname from `spec.controlPlane.gateway.hostname` +- Reference the same parent Gateway resource via `parentRefs` +- The `port` and `sectionName` fields are set automatically by Kamaji +- Route to the appropriate Tenant Control Plane service + +The Gateway resource must have listeners configured for both ports (6443 and 8132) to support both routes. + +## Prerequisites + +Before using Gateway API support, ensure: + +1. **Gateway API CRDs are installed** in your cluster (Required CRDs: `GatewayClass`, `Gateway`, `TLSRoute`) + +2. **A Gateway resource exists** with appropriate listeners configured: + - At minimum, listeners for ports 6443 (control plane) and 8132 (Konnectivity) + - TLS protocol with Passthrough mode + - Hostname pattern matching your Tenant Control Plane hostnames + +3. **DNS is configured** to resolve your hostnames to the Gateway's external address + +4. **Gateway controller is running** (e.g., Envoy Gateway, Istio Gateway, etc.) + +## Configuration + +### TenantControlPlane Gateway Configuration + +Enable Gateway API mode by setting the `spec.controlPlane.gateway` field in your TenantControlPlane resource: + +```yaml +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: tcp-1 +spec: + controlPlane: + # ... gateway configuration: + gateway: + hostname: "tcp1.cluster.dev" + parentRefs: + - name: gateway + namespace: default + additionalMetadata: + labels: + environment: production + annotations: + example.com/custom: "value" + # ... rest of the spec + deployment: + replicas: 1 + service: + serviceType: ClusterIP + dataStore: default + kubernetes: + version: v1.29.0 + kubelet: + cgroupfs: systemd + networkProfile: + port: 6443 + certSANs: + - "c11.cluster.dev" # make sure to set this. + addons: + coreDNS: {} + kubeProxy: {} + konnectivity: {} + +``` + +**Required fields:** + +- `hostname`: The hostname that will be used for routing (must match Gateway listener hostname pattern) +- `parentRefs`: Array of Gateway references (name and namespace) + +**Optional fields:** + +- `additionalMetadata.labels`: Custom labels to add to TLSRoute resources +- `additionalMetadata.annotations`: Custom annotations to add to TLSRoute resources + +!!! warning "Port and sectionName are set automatically" + Do not specify `port` or `sectionName` in `parentRefs`. Kamaji automatically sets these fields in TLSRoutes. + +### Gateway Resource Setup + +Your Gateway resource must have listeners configured for both the control plane and Konnectivity ports. Here's an example Gateway configuration: + +```yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: envoy-gw-class +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: gateway + namespace: default +spec: + gatewayClassName: envoy-gw-class + listeners: + - name: kube-apiserver + port: 6443 + protocol: TLS + hostname: 'tcp1.cluster.dev' + tls: + mode: Passthrough + allowedRoutes: + kinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + namespaces: + from: All + + # if konnectivity addon is enabled: + - name: konnectivity-server + port: 8132 + protocol: TLS + hostname: 'tcp1.cluster.dev' + tls: + mode: Passthrough + allowedRoutes: + kinds: + - group: gateway.networking.k8s.io + kind: TLSRoute + namespaces: + from: All + +``` + + +## Multiple Tenant Control Planes + +You can use the same Gateway resource for multiple Tenant Control Planes by using different hostnames: + +```yaml +# Gateway with wildcard hostname +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: gateway +spec: + listeners: + - hostname: '*.cluster.dev' + name: kube-apiserver + port: 6443 + # ... +--- +# Tenant Control Plane 1 +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: tcp-1 +spec: + controlPlane: + gateway: + hostname: "tcp1.cluster.dev" + parentRefs: + - name: gateway + namespace: default + # ... +--- +# Tenant Control Plane 2 +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: tcp-2 +spec: + controlPlane: + gateway: + hostname: "tcp2.cluster.dev" + parentRefs: + - name: gateway + namespace: default + # ... +``` + +Each Tenant Control Plane will get its own TLSRoutes with the respective hostnames, all routing through the same Gateway resource. + +You can check the Gateway status in the TenantControlPlane: + +```bash +kubectl get tenantcontrolplane tcp-1 -o yaml +``` + +Look for the `status.kubernetesResources.gateway` and `status.addons.konnectivity.gateway` fields. + + +## Additional Resources + +- [Gateway API Documentation](https://gateway-api.sigs.k8s.io/) +- [Quickstart with Envoy Gateway](https://gateway.envoyproxy.io/docs/tasks/quickstart/) + + diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index a7adcac..1670478 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -42688,6 +42688,13 @@ KonnectivityStatus defines the status of Konnectivity as Addon.
false + + gateway + object + + KubernetesGatewayStatus defines the status for the Tenant Control Plane Gateway in the management cluster.
+ + false kubeconfig object @@ -42875,6 +42882,505 @@ CertificatePrivateKeyPairStatus defines the status. +`TenantControlPlane.status.addons.konnectivity.gateway` + + +KubernetesGatewayStatus defines the status for the Tenant Control Plane Gateway in the management cluster. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
parents[]object + Parents is a list of parent resources (usually Gateways) that are +associated with the route, and the status of the route with respect to +each parent. When this route attaches to a parent, the controller that +manages the parent must add an entry to this list when the controller +first sees the route and should update the entry as appropriate when the +route or gateway is modified. + +Note that parent references that cannot be resolved by an implementation +of this API will not be added to this list. Implementations of this API +can only populate Route status for the Gateways/parent resources they are +responsible for. + +A maximum of 32 Gateways will be represented in this list. An empty list +means the route has not been attached to any Gateway. + + +Notes for implementors: + +While parents is not a listType `map`, this is due to the fact that the +list key is not scalar, and Kubernetes is unable to represent this. + +Parent status MUST be considered to be namespaced by the combination of +the parentRef and controllerName fields, and implementations should keep +the following rules in mind when updating this status: + +* Implementations MUST update only entries that have a matching value of + `controllerName` for that implementation. +* Implementations MUST NOT update entries with non-matching `controllerName` + fields. +* Implementations MUST treat each `parentRef`` in the Route separately and + update its status based on the relationship with that parent. +* Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + +
+ 		true
accessPoints[]object + A list of valid access points that the route exposes.
+ 		false
routeRefobject + Reference to the route created for this tenant.
+ 		false
+ + +`TenantControlPlane.status.addons.konnectivity.gateway.parents[index]` + + +RouteParentStatus describes the status of a route with respect to an +associated Parent. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
conditions[]object + Conditions describes the status of the route with respect to the Gateway. +Note that the route's availability is also subject to the Gateway's own +status conditions and listener status. + +If the Route's ParentRef specifies an existing Gateway that supports +Routes of this kind AND that Gateway's controller has sufficient access, +then that Gateway's controller MUST set the "Accepted" condition on the +Route, to indicate whether the route has been accepted or rejected by the +Gateway, and why. + +A Route MUST be considered "Accepted" if at least one of the Route's +rules is implemented by the Gateway. + +There are a number of cases where the "Accepted" condition may not be set +due to lack of controller visibility, that includes when: + +* The Route refers to a nonexistent parent. +* The Route is of a type that the controller does not support. +* The Route is in a namespace the controller does not have access to. + + + +Notes for implementors: + +Conditions are a listType `map`, which means that they function like a +map with a key of the `type` field _in the k8s apiserver_. + +This means that implementations must obey some rules when updating this +section. + +* Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. +* Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. +* Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. +* Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. +* If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + +
+ 		true
controllerNamestring + ControllerName is a domain/path string that indicates the name of the +controller that wrote this status. This corresponds with the +controllerName field on GatewayClass. + +Example: "example.net/gateway-controller". + +The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are +valid Kubernetes names +(https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + +Controllers MUST populate this field when writing status. Controllers should ensure that +entries to status populated with their ControllerName are cleaned up when they are no +longer necessary.
+ 		true
parentRefobject + ParentRef corresponds with a ParentRef in the spec that this +RouteParentStatus struct describes the status of.
+ 		true
+ + +`TenantControlPlane.status.addons.konnectivity.gateway.parents[index].conditions[index]` + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
+
+ Format: date-time
+ 		true
messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
+ 		true
reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
+ 		true
statusenum + status of the condition, one of True, False, Unknown.
+
+ Enum: True, False, Unknown
+ 		true
typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
+ 		true
observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
+
+ Format: int64
+ Minimum: 0
+ 		false
+ + +`TenantControlPlane.status.addons.konnectivity.gateway.parents[index].parentRef` + + +ParentRef corresponds with a ParentRef in the spec that this +RouteParentStatus struct describes the status of. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name is the name of the referent. + +Support: Core
+ 		true
groupstring + Group is the group of the referent. +When unspecified, "gateway.networking.k8s.io" is inferred. +To set the core API group (such as for a "Service" kind referent), +Group must be explicitly set to "" (empty string). + +Support: Core
+
+ Default: gateway.networking.k8s.io
+ 		false
kindstring + Kind is kind of the referent. + +There are two kinds of parent resources with "Core" support: + +* Gateway (Gateway conformance profile) +* Service (Mesh conformance profile, ClusterIP Services only) + +Support for other resources is Implementation-Specific.
+
+ Default: Gateway
+ 		false
namespacestring + Namespace is the namespace of the referent. When unspecified, this refers +to the local namespace of the Route. + +Note that there are specific rules for ParentRefs which cross namespace +boundaries. Cross-namespace references are only valid if they are explicitly +allowed by something in the namespace they are referring to. For example: +Gateway has the AllowedRoutes field, and ReferenceGrant provides a +generic way to enable any other kind of cross-namespace reference. + + +ParentRefs from a Route to a Service in the same namespace are "producer" +routes, which apply default routing rules to inbound connections from +any namespace to the Service. + +ParentRefs from a Route to a Service in a different namespace are +"consumer" routes, and these routing rules are only applied to outbound +connections originating from the same namespace as the Route, for which +the intended destination of the connections are a Service targeted as a +ParentRef of the Route. + + +Support: Core
+ 		false
portinteger + Port is the network port this Route targets. It can be interpreted +differently based on the type of parent resource. + +When the parent resource is a Gateway, this targets all listeners +listening on the specified port that also support this kind of Route(and +select this Route). It's not recommended to set `Port` unless the +networking behaviors specified in a Route must apply to a specific port +as opposed to a listener(s) whose port(s) may be changed. When both Port +and SectionName are specified, the name and port of the selected listener +must match both specified values. + + +When the parent resource is a Service, this targets a specific port in the +Service spec. When both Port (experimental) and SectionName are specified, +the name and port of the selected port must match both specified values. + + +Implementations MAY choose to support other parent resources. +Implementations supporting other types of parent resources MUST clearly +document how/if Port is interpreted. + +For the purpose of status, an attachment is considered successful as +long as the parent resource accepts it partially. For example, Gateway +listeners can restrict which Routes can attach to them by Route kind, +namespace, or hostname. If 1 of 2 Gateway listeners accept attachment +from the referencing Route, the Route MUST be considered successfully +attached. If no Gateway listeners accept attachment from this Route, +the Route MUST be considered detached from the Gateway. + +Support: Extended
+
+ Format: int32
+ Minimum: 1
+ Maximum: 65535
+ 		false
sectionNamestring + SectionName is the name of a section within the target resource. In the +following resources, SectionName is interpreted as the following: + +* Gateway: Listener name. When both Port (experimental) and SectionName +are specified, the name and port of the selected listener must match +both specified values. +* Service: Port name. When both Port (experimental) and SectionName +are specified, the name and port of the selected listener must match +both specified values. + +Implementations MAY choose to support attaching Routes to other resources. +If that is the case, they MUST clearly document how SectionName is +interpreted. + +When unspecified (empty string), this will reference the entire resource. +For the purpose of status, an attachment is considered successful if at +least one section in the parent resource accepts it. For example, Gateway +listeners can restrict which Routes can attach to them by Route kind, +namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from +the referencing Route, the Route MUST be considered successfully +attached. If no Gateway listeners accept attachment from this Route, the +Route MUST be considered detached from the Gateway. + +Support: Core
+ 		false
+ + +`TenantControlPlane.status.addons.konnectivity.gateway.accessPoints[index]` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
portinteger +
+
+ Format: int32
+ 		true
typestring + AddressType defines how a network address is represented as a text string. +This may take two possible forms: + +* A predefined CamelCase string identifier (currently limited to `IPAddress` or `Hostname`) +* A domain-prefixed string identifier (like `acme.io/CustomAddressType`) + +Values `IPAddress` and `Hostname` have Extended support. + +The `NamedAddress` value has been deprecated in favor of implementation +specific domain-prefixed strings. + +All other values, including domain-prefixed values have Implementation-specific support, +which are used in implementation-specific behaviors. Support for additional +predefined CamelCase identifiers may be added in future releases.
+ 		true
valuestring +
+ 		true
urls[]string +
+ 		false
+ + +`TenantControlPlane.status.addons.konnectivity.gateway.routeRef` + + +Reference to the route created for this tenant. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the referent. +This field is effectively required, but due to backwards compatibility is +allowed to be empty. Instances of this type with an empty value here are +almost certainly wrong. +More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+
+ Default:
+ 		false
+ + `TenantControlPlane.status.addons.konnectivity.kubeconfig` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 6326092..fe82264 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -80,6 +80,7 @@ nav: - guides/gitops.md - guides/console.md - guides/kubeconfig-generator.md + - guides/gateway-api.md - guides/upgrade.md - guides/monitoring.md - guides/terraform.md diff --git a/e2e/suite_test.go b/e2e/suite_test.go index 7346b91..9d16df8 100644 --- a/e2e/suite_test.go +++ b/e2e/suite_test.go @@ -4,10 +4,12 @@ package e2e import ( + "context" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" pointer "k8s.io/utils/ptr" @@ -68,9 +70,39 @@ var _ = BeforeSuite(func() { k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) + + By("creating GatewayClass for Gateway API tests") + gatewayClass := &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "envoy-gw-class", + }, + Spec: gatewayv1.GatewayClassSpec{ + ControllerName: "gateway.envoyproxy.io/gatewayclass-controller", + }, + } + Expect(k8sClient.Create(context.Background(), gatewayClass)).NotTo(HaveOccurred()) + + By("creating Gateway with kube-apiserver and konnectivity-server listeners") + CreateGatewayWithListeners("test-gateway", "default", "envoy-gw-class", "*.example.com") }) var _ = AfterSuite(func() { + By("deleting Gateway resources") + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gateway", + Namespace: "default", + }, + } + _ = k8sClient.Delete(context.Background(), gateway) + + gatewayClass := &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "envoy-gw-class", + }, + } + _ = k8sClient.Delete(context.Background(), gatewayClass) + By("tearing down the test environment") err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) diff --git a/e2e/tcp_gateway_konnectivity_ready_test.go b/e2e/tcp_gateway_konnectivity_ready_test.go new file mode 100644 index 0000000..28bfb27 --- /dev/null +++ b/e2e/tcp_gateway_konnectivity_ready_test.go @@ -0,0 +1,151 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + pointer "k8s.io/utils/ptr" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +var _ = Describe("Deploy a TenantControlPlane with Gateway API and Konnectivity", func() { + var tcp *kamajiv1alpha1.TenantControlPlane + + JustBeforeEach(func() { + tcp = &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-konnectivity-gateway", + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{ + ControlPlane: kamajiv1alpha1.ControlPlane{ + Deployment: kamajiv1alpha1.DeploymentSpec{ + Replicas: pointer.To(int32(1)), + }, + Service: kamajiv1alpha1.ServiceSpec{ + ServiceType: "ClusterIP", + }, + Gateway: &kamajiv1alpha1.GatewaySpec{ + Hostname: gatewayv1.Hostname("tcp-gateway-konnectivity.example.com"), + AdditionalMetadata: kamajiv1alpha1.AdditionalMetadata{ + Labels: map[string]string{ + "test.kamaji.io/gateway": "true", + }, + Annotations: map[string]string{ + "test.kamaji.io/created-by": "e2e-test", + }, + }, + GatewayParentRefs: []gatewayv1.ParentReference{ + { + Name: "test-gateway", + }, + }, + }, + }, + NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{ + Address: "172.18.0.4", + }, + Kubernetes: kamajiv1alpha1.KubernetesSpec{ + Version: "v1.28.0", + Kubelet: kamajiv1alpha1.KubeletSpec{ + CGroupFS: "cgroupfs", + }, + AdmissionControllers: kamajiv1alpha1.AdmissionControllers{ + "LimitRanger", + "ResourceQuota", + }, + }, + Addons: kamajiv1alpha1.AddonsSpec{ + Konnectivity: &kamajiv1alpha1.KonnectivitySpec{ + KonnectivityServerSpec: kamajiv1alpha1.KonnectivityServerSpec{ + Port: 8132, + }, + }, + }, + }, + } + Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred()) + }) + + JustAfterEach(func() { + Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed()) + + // Wait for the object to be completely deleted + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{ + Name: tcp.Name, + Namespace: tcp.Namespace, + }, &kamajiv1alpha1.TenantControlPlane{}) + + return err != nil // Returns true when object is not found (deleted) + }).WithTimeout(time.Minute).Should(BeTrue()) + }) + + It("Should be Ready", func() { + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) + }) + + It("Should create Konnectivity TLSRoute with correct sectionName", func() { + Eventually(func() error { + route := &gatewayv1alpha2.TLSRoute{} + if err := k8sClient.Get(context.Background(), types.NamespacedName{ + Name: tcp.Name + "-konnectivity", + Namespace: tcp.Namespace, + }, route); err != nil { + return err + } + if len(route.Spec.ParentRefs) == 0 { + return fmt.Errorf("parentRefs is empty") + } + if route.Spec.ParentRefs[0].SectionName == nil { + return fmt.Errorf("sectionName is nil") + } + if *route.Spec.ParentRefs[0].SectionName != gatewayv1.SectionName("konnectivity-server") { + return fmt.Errorf("expected sectionName 'konnectivity-server', got '%s'", *route.Spec.ParentRefs[0].SectionName) + } + + return nil + }).WithTimeout(time.Minute).Should(Succeed()) + }) + + It("Should use same hostname for both TLSRoutes", func() { + Eventually(func() error { + controlPlaneRoute := &gatewayv1alpha2.TLSRoute{} + if err := k8sClient.Get(context.Background(), types.NamespacedName{ + Name: tcp.Name, + Namespace: tcp.Namespace, + }, controlPlaneRoute); err != nil { + return err + } + + konnectivityRoute := &gatewayv1alpha2.TLSRoute{} + if err := k8sClient.Get(context.Background(), types.NamespacedName{ + Name: tcp.Name + "-konnectivity", + Namespace: tcp.Namespace, + }, konnectivityRoute); err != nil { + return err + } + + if len(controlPlaneRoute.Spec.Hostnames) == 0 || len(konnectivityRoute.Spec.Hostnames) == 0 { + return fmt.Errorf("hostnames are empty") + } + if controlPlaneRoute.Spec.Hostnames[0] != konnectivityRoute.Spec.Hostnames[0] { + return fmt.Errorf("hostnames do not match: control plane '%s', konnectivity '%s'", + controlPlaneRoute.Spec.Hostnames[0], konnectivityRoute.Spec.Hostnames[0]) + } + + return nil + }).WithTimeout(time.Minute).Should(Succeed()) + }) +}) diff --git a/e2e/tcp_gateway_ready_test.go b/e2e/tcp_gateway_ready_test.go index 5cf3ff9..a819db7 100644 --- a/e2e/tcp_gateway_ready_test.go +++ b/e2e/tcp_gateway_ready_test.go @@ -5,6 +5,7 @@ package e2e import ( "context" + "fmt" "time" . "github.com/onsi/ginkgo/v2" @@ -89,14 +90,39 @@ var _ = Describe("Deploy a TenantControlPlane with Gateway API", func() { StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) }) - It("Should create TLSRoute resource", func() { + It("Should create control plane TLSRoute with correct sectionName", func() { Eventually(func() error { route := &gatewayv1alpha2.TLSRoute{} // TODO: Check ownership. - return k8sClient.Get(context.Background(), types.NamespacedName{ + if err := k8sClient.Get(context.Background(), types.NamespacedName{ Name: tcp.Name, Namespace: tcp.Namespace, - }, route) + }, route); err != nil { + return err + } + if len(route.Spec.ParentRefs) == 0 { + return fmt.Errorf("parentRefs is empty") + } + if route.Spec.ParentRefs[0].SectionName == nil { + return fmt.Errorf("sectionName is nil") + } + if *route.Spec.ParentRefs[0].SectionName != gatewayv1.SectionName("kube-apiserver") { + return fmt.Errorf("expected sectionName 'kube-apiserver', got '%s'", *route.Spec.ParentRefs[0].SectionName) + } + + return nil }).WithTimeout(time.Minute).Should(Succeed()) }) + + It("Should not create Konnectivity TLSRoute", func() { + // Verify Konnectivity route is not created + Consistently(func() error { + route := &gatewayv1alpha2.TLSRoute{} + + return k8sClient.Get(context.Background(), types.NamespacedName{ + Name: tcp.Name + "-konnectivity", + Namespace: tcp.Namespace, + }, route) + }, 10*time.Second, time.Second).Should(HaveOccurred()) + }) }) diff --git a/e2e/utils_test.go b/e2e/utils_test.go index 8a81a44..be1053e 100644 --- a/e2e/utils_test.go +++ b/e2e/utils_test.go @@ -19,7 +19,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" + pointer "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" ) @@ -209,3 +211,60 @@ func ScaleTenantControlPlane(tcp *kamajiv1alpha1.TenantControlPlane, replicas in }) Expect(err).To(Succeed()) } + +// CreateGatewayWithListeners creates a Gateway with both kube-apiserver and konnectivity-server listeners. +func CreateGatewayWithListeners(gatewayName, namespace, gatewayClassName, hostname string) { + GinkgoHelper() + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: gatewayName, + Namespace: namespace, + }, + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: gatewayv1.ObjectName(gatewayClassName), + Listeners: []gatewayv1.Listener{ + { + Name: "kube-apiserver", + Port: 6443, + Protocol: gatewayv1.TLSProtocolType, + Hostname: pointer.To(gatewayv1.Hostname(hostname)), + TLS: &gatewayv1.ListenerTLSConfig{ + Mode: pointer.To(gatewayv1.TLSModeType("Passthrough")), + }, + AllowedRoutes: &gatewayv1.AllowedRoutes{ + Namespaces: &gatewayv1.RouteNamespaces{ + From: pointer.To(gatewayv1.NamespacesFromAll), + }, + Kinds: []gatewayv1.RouteGroupKind{ + { + Group: pointer.To(gatewayv1.Group("gateway.networking.k8s.io")), + Kind: "TLSRoute", + }, + }, + }, + }, + { + Name: "konnectivity-server", + Port: 8132, + Protocol: gatewayv1.TLSProtocolType, + Hostname: pointer.To(gatewayv1.Hostname(hostname)), + TLS: &gatewayv1.ListenerTLSConfig{ + Mode: pointer.To(gatewayv1.TLSModeType("Passthrough")), + }, + AllowedRoutes: &gatewayv1.AllowedRoutes{ + Namespaces: &gatewayv1.RouteNamespaces{ + From: pointer.To(gatewayv1.NamespacesFromAll), + }, + Kinds: []gatewayv1.RouteGroupKind{ + { + Group: pointer.To(gatewayv1.Group("gateway.networking.k8s.io")), + Kind: "TLSRoute", + }, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(context.Background(), gateway)).NotTo(HaveOccurred()) +} diff --git a/internal/resources/k8s_gateway_resource.go b/internal/resources/k8s_gateway_resource.go index 7def916..fccdddd 100644 --- a/internal/resources/k8s_gateway_resource.go +++ b/internal/resources/k8s_gateway_resource.go @@ -6,14 +6,10 @@ package resources import ( "context" "fmt" - "net/url" "github.com/prometheus/client_golang/prometheus" v1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" @@ -56,66 +52,12 @@ func (r *KubernetesGatewayResource) gatewayStatusNeedsUpdate(tcp *kamajiv1alpha1 currentStatus := tcp.Status.Kubernetes.Gateway // Check if route reference has changed - if currentStatus.RouteRef.Name != r.resource.Name { + if currentStatus != nil && currentStatus.RouteRef.Name != r.resource.Name { return true } // Compare RouteStatus - check if number of parents changed - if len(currentStatus.RouteStatus.Parents) != len(r.resource.Status.RouteStatus.Parents) { - return true - } - - // Compare individual parent statuses - // NOTE: Multiple Parent References are assumed. - for i, currentParent := range currentStatus.RouteStatus.Parents { - if i >= len(r.resource.Status.RouteStatus.Parents) { - return true - } - - resourceParent := r.resource.Status.RouteStatus.Parents[i] - - // Compare parent references - if currentParent.ParentRef.Name != resourceParent.ParentRef.Name || - (currentParent.ParentRef.Namespace == nil) != (resourceParent.ParentRef.Namespace == nil) || - (currentParent.ParentRef.Namespace != nil && resourceParent.ParentRef.Namespace != nil && - *currentParent.ParentRef.Namespace != *resourceParent.ParentRef.Namespace) || - (currentParent.ParentRef.SectionName == nil) != (resourceParent.ParentRef.SectionName == nil) || - (currentParent.ParentRef.SectionName != nil && resourceParent.ParentRef.SectionName != nil && - *currentParent.ParentRef.SectionName != *resourceParent.ParentRef.SectionName) { - return true - } - - if len(currentParent.Conditions) != len(resourceParent.Conditions) { - return true - } - - // Compare each condition - for j, currentCondition := range currentParent.Conditions { - if j >= len(resourceParent.Conditions) { - return true - } - - resourceCondition := resourceParent.Conditions[j] - - if currentCondition.Type != resourceCondition.Type || - currentCondition.Status != resourceCondition.Status || - currentCondition.Reason != resourceCondition.Reason || - currentCondition.Message != resourceCondition.Message || - !currentCondition.LastTransitionTime.Equal(&resourceCondition.LastTransitionTime) { - return true - } - } - } - - // Since access points are derived from route status and gateway conditions, - // and we've already compared the route status above, we can assume that - // if the route status hasn't changed, the access points calculation - // will produce the same result. This avoids the need for complex - // gateway fetching in the status comparison. - // - // If there are edge cases where gateway state changes but route status doesn't, - // those will be caught in the next reconciliation cycle anyway. - return false + return IsGatewayRouteStatusChanged(currentStatus, r.resource.Status.RouteStatus) } func (r *KubernetesGatewayResource) ShouldCleanup(tcp *kamajiv1alpha1.TenantControlPlane) bool { @@ -125,95 +67,18 @@ func (r *KubernetesGatewayResource) ShouldCleanup(tcp *kamajiv1alpha1.TenantCont func (r *KubernetesGatewayResource) CleanUp(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (bool, error) { logger := log.FromContext(ctx, "resource", r.GetName()) - route := gatewayv1alpha2.TLSRoute{} - if err := r.Client.Get(ctx, client.ObjectKey{ - Namespace: r.resource.GetNamespace(), - Name: r.resource.GetName(), - }, &route); err != nil { - if !k8serrors.IsNotFound(err) { - logger.Error(err, "failed to get TLSRoute before cleanup") + cleaned, err := CleanupTLSRoute(ctx, r.Client, r.resource.GetName(), r.resource.GetNamespace(), tcp) + if err != nil { + logger.Error(err, "failed to cleanup tcp route") - return false, err - } - - return false, nil + return false, err } - if !metav1.IsControlledBy(&route, tcp) { - logger.Info("skipping cleanup: route is not managed by Kamaji", "name", route.Name, "namespace", route.Namespace) - - return false, nil + if cleaned { + logger.V(1).Info("tcp route cleaned up successfully") } - if err := r.Client.Delete(ctx, &route); err != nil { - if !k8serrors.IsNotFound(err) { - // TODO: Is that an error? Wanted to delete the resource anyways. - logger.Error(err, "cannot cleanup tcp route") - - return false, err - } - - return false, nil - } - - logger.V(1).Info("tcp route cleaned up successfully") - - return true, nil -} - -// fetchGatewayByListener uses the indexer to efficiently find a gateway with a specific listener. -// This avoids the need to iterate through all listeners in a gateway. -func (r *KubernetesGatewayResource) fetchGatewayByListener(ctx context.Context, ref gatewayv1.ParentReference) (*gatewayv1.Gateway, error) { - if ref.Namespace == nil { - return nil, fmt.Errorf("missing namespace") - } - if ref.SectionName == nil { - return nil, fmt.Errorf("missing sectionName") - } - - // Build the composite key that matches our indexer format: namespace/gatewayName/listenerName - listenerKey := fmt.Sprintf("%s/%s/%s", *ref.Namespace, ref.Name, *ref.SectionName) - - // Query gateways using the indexer - gatewayList := &gatewayv1.GatewayList{} - if err := r.Client.List(ctx, gatewayList, client.MatchingFieldsSelector{ - Selector: fields.OneTermEqualSelector(kamajiv1alpha1.GatewayListenerNameKey, listenerKey), - }); err != nil { - return nil, fmt.Errorf("failed to list gateways by listener: %w", err) - } - - if len(gatewayList.Items) == 0 { - return nil, fmt.Errorf("no gateway found with listener '%s'", *ref.SectionName) - } - - // Since we're using a composite key with namespace/name/listener, we should get exactly one result - if len(gatewayList.Items) > 1 { - return nil, fmt.Errorf("found multiple gateways with listener '%s', expected exactly one", *ref.SectionName) - } - - return &gatewayList.Items[0], nil -} - -func FindMatchingListener(listeners []gatewayv1.Listener, ref gatewayv1.ParentReference) (gatewayv1.Listener, error) { - if ref.SectionName == nil { - return gatewayv1.Listener{}, fmt.Errorf("missing sectionName") - } - name := *ref.SectionName - for _, listener := range listeners { - if listener.Name == name { - return listener, nil - } - } - - // TODO: Handle the cases according to the spec: - // - When both Port (experimental) and SectionName are - // specified, the name and port of the selected listener - // must match both specified values. - // - When unspecified (empty string) this will reference - // the entire resource [...] an attachment is considered - // successful if at least one section in the parent resource accepts it - - return gatewayv1.Listener{}, fmt.Errorf("could not find listener '%s'", name) + return cleaned, nil } func (r *KubernetesGatewayResource) UpdateTenantControlPlaneStatus(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error { @@ -251,53 +116,9 @@ func (r *KubernetesGatewayResource) UpdateTenantControlPlaneStatus(ctx context.C } logger.V(1).Info("updating TenantControlPlane status for Gateway routes") - accessPoints := []kamajiv1alpha1.GatewayAccessPoint{} - for _, routeStatus := range routeStatuses.Parents { - routeAccepted := meta.IsStatusConditionTrue( - routeStatus.Conditions, - string(gatewayv1.RouteConditionAccepted), - ) - if !routeAccepted { - continue - } - - // Use the indexer to efficiently find the gateway with the specific listener - gateway, err := r.fetchGatewayByListener(ctx, routeStatus.ParentRef) - if err != nil { - return fmt.Errorf("could not fetch gateway with listener '%v': %w", - routeStatus.ParentRef.SectionName, err) - } - gatewayProgrammed := meta.IsStatusConditionTrue( - gateway.Status.Conditions, - string(gatewayv1.GatewayConditionProgrammed), - ) - if !gatewayProgrammed { - continue - } - - // Since we fetched the gateway using the indexer, we know the listener exists - // but we still need to get its details from the gateway spec - listener, err := FindMatchingListener( - gateway.Spec.Listeners, routeStatus.ParentRef, - ) - if err != nil { - return fmt.Errorf("failed to match listener: %w", err) - } - - for _, hostname := range r.resource.Spec.Hostnames { - rawURL := fmt.Sprintf("https://%s:%d", hostname, listener.Port) - url, err := url.Parse(rawURL) - if err != nil { - return fmt.Errorf("invalid url: %w", err) - } - - hostnameAddressType := gatewayv1.HostnameAddressType - accessPoints = append(accessPoints, kamajiv1alpha1.GatewayAccessPoint{ - Type: &hostnameAddressType, - Value: url.String(), - Port: listener.Port, - }) - } + accessPoints, err := BuildGatewayAccessPointsStatus(ctx, r.Client, r.resource, routeStatuses) + if err != nil { + return err } tcp.Status.Kubernetes.Gateway.AccessPoints = accessPoints @@ -329,10 +150,6 @@ func (r *KubernetesGatewayResource) mutate(tcp *kamajiv1alpha1.TenantControlPlan tcp.Spec.ControlPlane.Gateway.AdditionalMetadata.Annotations) r.resource.SetAnnotations(annotations) - if tcp.Spec.ControlPlane.Gateway.GatewayParentRefs != nil { - r.resource.Spec.ParentRefs = tcp.Spec.ControlPlane.Gateway.GatewayParentRefs - } - serviceName := gatewayv1alpha2.ObjectName(tcp.Status.Kubernetes.Service.Name) servicePort := tcp.Status.Kubernetes.Service.Port @@ -340,6 +157,11 @@ func (r *KubernetesGatewayResource) mutate(tcp *kamajiv1alpha1.TenantControlPlan return fmt.Errorf("service not ready, cannot create TLSRoute") } + if tcp.Spec.ControlPlane.Gateway.GatewayParentRefs != nil { + // Copy parentRefs and explicitly set port and sectionName fields + r.resource.Spec.ParentRefs = NewParentRefsSpecWithPortAndSection(tcp.Spec.ControlPlane.Gateway.GatewayParentRefs, servicePort, "kube-apiserver") + } + rule := gatewayv1alpha2.TLSRouteRule{ BackendRefs: []gatewayv1alpha2.BackendRef{ { diff --git a/internal/resources/k8s_gateway_resource_test.go b/internal/resources/k8s_gateway_resource_test.go index 514def8..e12ef41 100644 --- a/internal/resources/k8s_gateway_resource_test.go +++ b/internal/resources/k8s_gateway_resource_test.go @@ -12,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -100,6 +101,62 @@ var _ = Describe("KubernetesGatewayResource", func() { shouldUpdate := resource.ShouldStatusBeUpdated(ctx, tcp) Expect(shouldUpdate).To(BeTrue()) }) + + It("should set port and sectionName in parentRefs, overriding any user-provided values", func() { + customPort := gatewayv1.PortNumber(9999) + customSectionName := gatewayv1.SectionName("custom") + tcp.Spec.ControlPlane.Gateway.GatewayParentRefs[0].Port = &customPort + tcp.Spec.ControlPlane.Gateway.GatewayParentRefs[0].SectionName = &customSectionName + + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + _, err = resource.CreateOrUpdate(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + route := &gatewayv1alpha2.TLSRoute{} + err = resource.Client.Get(ctx, client.ObjectKey{Name: tcp.Name, Namespace: tcp.Namespace}, route) + Expect(err).NotTo(HaveOccurred()) + Expect(route.Spec.ParentRefs).To(HaveLen(1)) + Expect(route.Spec.ParentRefs[0].Port).NotTo(BeNil()) + Expect(*route.Spec.ParentRefs[0].Port).To(Equal(tcp.Status.Kubernetes.Service.Port)) + Expect(*route.Spec.ParentRefs[0].Port).NotTo(Equal(customPort)) + Expect(route.Spec.ParentRefs[0].SectionName).NotTo(BeNil()) + Expect(*route.Spec.ParentRefs[0].SectionName).To(Equal(gatewayv1.SectionName("kube-apiserver"))) + Expect(*route.Spec.ParentRefs[0].SectionName).NotTo(Equal(customSectionName)) + }) + + It("should handle multiple parentRefs correctly", func() { + namespace := gatewayv1.Namespace("default") + tcp.Spec.ControlPlane.Gateway.GatewayParentRefs = []gatewayv1alpha2.ParentReference{ + { + Name: "test-gateway-1", + Namespace: &namespace, + }, + { + Name: "test-gateway-2", + Namespace: &namespace, + }, + } + + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + _, err = resource.CreateOrUpdate(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + route := &gatewayv1alpha2.TLSRoute{} + err = resource.Client.Get(ctx, client.ObjectKey{Name: tcp.Name, Namespace: tcp.Namespace}, route) + Expect(err).NotTo(HaveOccurred()) + Expect(route.Spec.ParentRefs).To(HaveLen(2)) + + for i := range route.Spec.ParentRefs { + Expect(route.Spec.ParentRefs[i].Port).NotTo(BeNil()) + Expect(*route.Spec.ParentRefs[i].Port).To(Equal(tcp.Status.Kubernetes.Service.Port)) + Expect(route.Spec.ParentRefs[i].SectionName).NotTo(BeNil()) + Expect(*route.Spec.ParentRefs[i].SectionName).To(Equal(gatewayv1.SectionName("kube-apiserver"))) + } + }) }) Context("When GatewayRoutes is not configured", func() { @@ -235,4 +292,81 @@ var _ = Describe("KubernetesGatewayResource", func() { Expect(listener.Port).To(Equal(gatewayv1.PortNumber(80))) }) }) + + Describe("NewParentRefsSpecWithPortAndSection", func() { + var ( + parentRefs []gatewayv1.ParentReference + testPort int32 + testSectionName string + ) + + BeforeEach(func() { + namespace := gatewayv1.Namespace("default") + namespace2 := gatewayv1.Namespace("other") + testPort = int32(6443) + testSectionName = "kube-apiserver" + originalPort := gatewayv1.PortNumber(9999) + originalSectionName := gatewayv1.SectionName("original") + parentRefs = []gatewayv1.ParentReference{ + { + Name: "test-gateway-1", + Namespace: &namespace, + Port: &originalPort, + SectionName: &originalSectionName, + }, + { + Name: "test-gateway-2", + Namespace: &namespace2, + }, + } + }) + + It("should create copy of parentRefs with port and sectionName set", func() { + result := resources.NewParentRefsSpecWithPortAndSection(parentRefs, testPort, testSectionName) + + Expect(result).To(HaveLen(2)) + for i := range result { + Expect(result[i].Name).To(Equal(parentRefs[i].Name)) + Expect(result[i].Namespace).To(Equal(parentRefs[i].Namespace)) + Expect(result[i].Port).NotTo(BeNil()) + Expect(*result[i].Port).To(Equal(testPort)) + Expect(result[i].SectionName).NotTo(BeNil()) + Expect(*result[i].SectionName).To(Equal(gatewayv1.SectionName(testSectionName))) + } + }) + + It("should not modify original parentRefs", func() { + // Store original values for verification + originalFirstPort := parentRefs[0].Port + originalFirstSectionName := parentRefs[0].SectionName + originalSecondPort := parentRefs[1].Port + originalSecondSectionName := parentRefs[1].SectionName + + result := resources.NewParentRefsSpecWithPortAndSection(parentRefs, testPort, testSectionName) + + // Original should remain unchanged + Expect(parentRefs[0].Port).To(Equal(originalFirstPort)) + Expect(parentRefs[0].SectionName).To(Equal(originalFirstSectionName)) + Expect(parentRefs[1].Port).To(Equal(originalSecondPort)) + Expect(parentRefs[1].SectionName).To(Equal(originalSecondSectionName)) + + // Result should have new values + Expect(result[0].Port).NotTo(BeNil()) + Expect(*result[0].Port).To(Equal(testPort)) + Expect(result[0].SectionName).NotTo(BeNil()) + Expect(*result[0].SectionName).To(Equal(gatewayv1.SectionName(testSectionName))) + Expect(result[1].Port).NotTo(BeNil()) + Expect(*result[1].Port).To(Equal(testPort)) + Expect(result[1].SectionName).NotTo(BeNil()) + Expect(*result[1].SectionName).To(Equal(gatewayv1.SectionName(testSectionName))) + }) + + It("should handle empty parentRefs slice", func() { + parentRefs = []gatewayv1.ParentReference{} + + result := resources.NewParentRefsSpecWithPortAndSection(parentRefs, testPort, testSectionName) + + Expect(result).To(BeEmpty()) + }) + }) }) diff --git a/internal/resources/k8s_gateway_utils.go b/internal/resources/k8s_gateway_utils.go new file mode 100644 index 0000000..fc2b42d --- /dev/null +++ b/internal/resources/k8s_gateway_utils.go @@ -0,0 +1,241 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + "net/url" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +// fetchGatewayByListener uses the indexer to efficiently find a gateway with a specific listener. +// This avoids the need to iterate through all listeners in a gateway. +func fetchGatewayByListener(ctx context.Context, c client.Client, ref gatewayv1.ParentReference) (*gatewayv1.Gateway, error) { + if ref.SectionName == nil { + return nil, fmt.Errorf("missing sectionName") + } + + // Build the composite key that matches our indexer format: namespace/gatewayName/listenerName + listenerKey := fmt.Sprintf("%s/%s/%s", *ref.Namespace, ref.Name, *ref.SectionName) + + // Query gateways using the indexer + gatewayList := &gatewayv1.GatewayList{} + if err := c.List(ctx, gatewayList, client.MatchingFieldsSelector{ + Selector: fields.OneTermEqualSelector(kamajiv1alpha1.GatewayListenerNameKey, listenerKey), + }); err != nil { + return nil, fmt.Errorf("failed to list gateways by listener: %w", err) + } + + if len(gatewayList.Items) == 0 { + return nil, fmt.Errorf("no gateway found with listener '%s'", *ref.SectionName) + } + + // Since we're using a composite key with namespace/name/listener, we should get exactly one result + if len(gatewayList.Items) > 1 { + return nil, fmt.Errorf("found multiple gateways with listener '%s', expected exactly one", *ref.SectionName) + } + + return &gatewayList.Items[0], nil +} + +// FindMatchingListener finds a listener in the given list that matches the parent reference. +func FindMatchingListener(listeners []gatewayv1.Listener, ref gatewayv1.ParentReference) (gatewayv1.Listener, error) { + if ref.SectionName == nil { + return gatewayv1.Listener{}, fmt.Errorf("missing sectionName") + } + name := *ref.SectionName + for _, listener := range listeners { + if listener.Name == name { + return listener, nil + } + } + + // TODO: Handle the cases according to the spec: + // - When both Port (experimental) and SectionName are + // specified, the name and port of the selected listener + // must match both specified values. + // - When unspecified (empty string) this will reference + // the entire resource [...] an attachment is considered + // successful if at least one section in the parent resource accepts it + + return gatewayv1.Listener{}, fmt.Errorf("could not find listener '%s'", name) +} + +// IsGatewayRouteStatusChanged checks if the gateway route status has changed compared to the stored status. +// Returns true if the status has changed (update needed), false if it's the same. +func IsGatewayRouteStatusChanged(currentStatus *kamajiv1alpha1.KubernetesGatewayStatus, resourceStatus gatewayv1alpha2.RouteStatus) bool { + if currentStatus == nil { + return true + } + + // Compare RouteStatus - check if number of parents changed + if len(currentStatus.RouteStatus.Parents) != len(resourceStatus.Parents) { + return true + } + + // Compare individual parent statuses + // NOTE: Multiple Parent References are assumed. + for i, currentParent := range currentStatus.RouteStatus.Parents { + if i >= len(resourceStatus.Parents) { + return true + } + + resourceParent := resourceStatus.Parents[i] + + // Compare parent references + if currentParent.ParentRef.Name != resourceParent.ParentRef.Name || + (currentParent.ParentRef.Namespace == nil) != (resourceParent.ParentRef.Namespace == nil) || + (currentParent.ParentRef.Namespace != nil && resourceParent.ParentRef.Namespace != nil && + *currentParent.ParentRef.Namespace != *resourceParent.ParentRef.Namespace) || + (currentParent.ParentRef.SectionName == nil) != (resourceParent.ParentRef.SectionName == nil) || + (currentParent.ParentRef.SectionName != nil && resourceParent.ParentRef.SectionName != nil && + *currentParent.ParentRef.SectionName != *resourceParent.ParentRef.SectionName) { + return true + } + + if len(currentParent.Conditions) != len(resourceParent.Conditions) { + return true + } + + // Compare each condition + for j, currentCondition := range currentParent.Conditions { + if j >= len(resourceParent.Conditions) { + return true + } + + resourceCondition := resourceParent.Conditions[j] + + if currentCondition.Type != resourceCondition.Type || + currentCondition.Status != resourceCondition.Status || + currentCondition.Reason != resourceCondition.Reason || + currentCondition.Message != resourceCondition.Message || + !currentCondition.LastTransitionTime.Equal(&resourceCondition.LastTransitionTime) { + return true + } + } + } + + // Since access points are derived from route status and gateway conditions, + // and we've already compared the route status above, we can assume that + // if the route status hasn't changed, the access points calculation + // will produce the same result. This avoids the need for complex + // gateway fetching in the status comparison. + // + // If there are edge cases where gateway state changes but route status doesn't, + // those will be caught in the next reconciliation cycle anyway. + return false +} + +// CleanupTLSRoute cleans up a TLSRoute resource if it's managed by the given TenantControlPlane. +func CleanupTLSRoute(ctx context.Context, c client.Client, routeName, routeNamespace string, tcp metav1.Object) (bool, error) { + route := gatewayv1alpha2.TLSRoute{} + if err := c.Get(ctx, client.ObjectKey{ + Namespace: routeNamespace, + Name: routeName, + }, &route); err != nil { + if !k8serrors.IsNotFound(err) { + return false, fmt.Errorf("failed to get TLSRoute before cleanup: %w", err) + } + + return false, nil + } + + if !metav1.IsControlledBy(&route, tcp) { + return false, nil + } + + if err := c.Delete(ctx, &route); err != nil { + if !k8serrors.IsNotFound(err) { + return false, fmt.Errorf("cannot delete TLSRoute route: %w", err) + } + + return false, nil + } + + return true, nil +} + +// BuildGatewayAccessPointsStatus builds access points from route statuses. +func BuildGatewayAccessPointsStatus(ctx context.Context, c client.Client, route *gatewayv1alpha2.TLSRoute, routeStatuses gatewayv1alpha2.RouteStatus) ([]kamajiv1alpha1.GatewayAccessPoint, error) { + accessPoints := []kamajiv1alpha1.GatewayAccessPoint{} + routeNamespace := gatewayv1.Namespace(route.Namespace) + + for _, routeStatus := range routeStatuses.Parents { + routeAccepted := meta.IsStatusConditionTrue( + routeStatus.Conditions, + string(gatewayv1.RouteConditionAccepted), + ) + if !routeAccepted { + continue + } + + if routeStatus.ParentRef.Namespace == nil { + // Set the namespace to the route namespace if not set + routeStatus.ParentRef.Namespace = &routeNamespace + } + + // Use the indexer to efficiently find the gateway with the specific listener + gateway, err := fetchGatewayByListener(ctx, c, routeStatus.ParentRef) + if err != nil { + return nil, fmt.Errorf("could not fetch gateway with listener '%v': %w", + routeStatus.ParentRef.SectionName, err) + } + gatewayProgrammed := meta.IsStatusConditionTrue( + gateway.Status.Conditions, + string(gatewayv1.GatewayConditionProgrammed), + ) + if !gatewayProgrammed { + continue + } + + // Since we fetched the gateway using the indexer, we know the listener exists + // but we still need to get its details from the gateway spec + listener, err := FindMatchingListener( + gateway.Spec.Listeners, routeStatus.ParentRef, + ) + if err != nil { + return nil, fmt.Errorf("failed to match listener: %w", err) + } + + for _, hostname := range route.Spec.Hostnames { + rawURL := fmt.Sprintf("https://%s:%d", hostname, listener.Port) + parsedURL, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("invalid url: %w", err) + } + + hostnameAddressType := gatewayv1.HostnameAddressType + accessPoints = append(accessPoints, kamajiv1alpha1.GatewayAccessPoint{ + Type: &hostnameAddressType, + Value: parsedURL.String(), + Port: listener.Port, + }) + } + } + + return accessPoints, nil +} + +// NewParentRefsSpecWithPortAndSection creates a copy of parentRefs with port and sectionName set for each reference. +func NewParentRefsSpecWithPortAndSection(parentRefs []gatewayv1.ParentReference, port int32, sectionName string) []gatewayv1.ParentReference { + result := make([]gatewayv1.ParentReference, len(parentRefs)) + sectionNamePtr := gatewayv1.SectionName(sectionName) + for i, parentRef := range parentRefs { + result[i] = *parentRef.DeepCopy() + result[i].Port = &port + result[i].SectionName = §ionNamePtr + } + + return result +} diff --git a/internal/resources/konnectivity/agent.go b/internal/resources/konnectivity/agent.go index e2f5d19..8fe1857 100644 --- a/internal/resources/konnectivity/agent.go +++ b/internal/resources/konnectivity/agent.go @@ -182,6 +182,21 @@ func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.T return err } + // Override address with control plane gateway hostname if configured + // Konnectivity TLSRoute uses the same hostname as control plane gateway + if tenantControlPlane.Spec.ControlPlane.Gateway != nil && + len(tenantControlPlane.Spec.ControlPlane.Gateway.Hostname) > 0 { + hostname := tenantControlPlane.Spec.ControlPlane.Gateway.Hostname + + // Extract hostname + if len(hostname) > 0 { + konnectivityHostname, _ := utilities.GetControlPlaneAddressAndPortFromHostname( + string(hostname), + tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityServerSpec.Port) + address = konnectivityHostname + } + } + r.resource.SetLabels(utilities.MergeMaps(r.resource.GetLabels(), utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()))) specSelector := &metav1.LabelSelector{ diff --git a/internal/resources/konnectivity/gateway_resource.go b/internal/resources/konnectivity/gateway_resource.go new file mode 100644 index 0000000..87822e1 --- /dev/null +++ b/internal/resources/konnectivity/gateway_resource.go @@ -0,0 +1,232 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package konnectivity + +import ( + "context" + "fmt" + + "github.com/prometheus/client_golang/prometheus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/resources" + "github.com/clastix/kamaji/internal/utilities" +) + +type KubernetesKonnectivityGatewayResource struct { + resource *gatewayv1alpha2.TLSRoute + Client client.Client +} + +func (r *KubernetesKonnectivityGatewayResource) GetHistogram() prometheus.Histogram { + gatewayCollector = resources.LazyLoadHistogramFromResource(gatewayCollector, r) + + return gatewayCollector +} + +func (r *KubernetesKonnectivityGatewayResource) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool { + switch { + case !r.shouldHaveGateway(tcp) && (tcp.Status.Addons.Konnectivity.Gateway == nil): + return false + case r.shouldHaveGateway(tcp) && (tcp.Status.Addons.Konnectivity.Gateway == nil): + return true + case !r.shouldHaveGateway(tcp) && (tcp.Status.Addons.Konnectivity.Gateway != nil): + return true + case r.shouldHaveGateway(tcp) && (tcp.Status.Addons.Konnectivity.Gateway != nil): + return r.gatewayStatusNeedsUpdate(tcp) + } + + return false +} + +// shouldHaveGateway checks if Konnectivity gateway should be configured. +// Create when Konnectivity addon is enabled and control plane gateway is configured. +func (r *KubernetesKonnectivityGatewayResource) shouldHaveGateway(tcp *kamajiv1alpha1.TenantControlPlane) bool { + if tcp.Spec.Addons.Konnectivity == nil { // konnectivity addon is disabled + return false + } + // Create when control plane gateway is configured + return tcp.Spec.ControlPlane.Gateway != nil +} + +// gatewayStatusNeedsUpdate compares the current gateway resource status with the stored status. +func (r *KubernetesKonnectivityGatewayResource) gatewayStatusNeedsUpdate(tcp *kamajiv1alpha1.TenantControlPlane) bool { + currentStatus := tcp.Status.Addons.Konnectivity.Gateway + + // Check if route reference has changed + if currentStatus != nil && currentStatus.RouteRef.Name != r.resource.Name { + return true + } + + // Compare RouteStatus - check if number of parents changed + return resources.IsGatewayRouteStatusChanged(currentStatus, r.resource.Status.RouteStatus) +} + +func (r *KubernetesKonnectivityGatewayResource) ShouldCleanup(tcp *kamajiv1alpha1.TenantControlPlane) bool { + return !r.shouldHaveGateway(tcp) && tcp.Status.Addons.Konnectivity.Gateway != nil +} + +func (r *KubernetesKonnectivityGatewayResource) CleanUp(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (bool, error) { + logger := log.FromContext(ctx, "resource", r.GetName()) + + cleaned, err := resources.CleanupTLSRoute(ctx, r.Client, r.resource.GetName(), r.resource.GetNamespace(), tcp) + if err != nil { + logger.Error(err, "failed to cleanup konnectivity route") + + return false, err + } + + if cleaned { + logger.V(1).Info("konnectivity route cleaned up successfully") + } + + return cleaned, nil +} + +func (r *KubernetesKonnectivityGatewayResource) UpdateTenantControlPlaneStatus(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error { + logger := log.FromContext(ctx, "resource", r.GetName()) + + // Clean up status if Gateway routes are no longer configured + if !r.shouldHaveGateway(tcp) { + tcp.Status.Addons.Konnectivity.Gateway = nil + + return nil + } + + tcp.Status.Addons.Konnectivity.Gateway = &kamajiv1alpha1.KubernetesGatewayStatus{ + RouteStatus: r.resource.Status.RouteStatus, + RouteRef: v1.LocalObjectReference{ + Name: r.resource.Name, + }, + } + + routeStatuses := tcp.Status.Addons.Konnectivity.Gateway.RouteStatus + + // TODO: Investigate the implications of having multiple parents / hostnames + // TODO: Use condition to report? + if len(routeStatuses.Parents) == 0 { + return fmt.Errorf("no gateway attached to the konnectivity route") + } + if len(routeStatuses.Parents) > 1 { + return fmt.Errorf("too many gateways attached to the konnectivity route") + } + if len(r.resource.Spec.Hostnames) == 0 { + return fmt.Errorf("no hostname in the konnectivity route") + } + if len(r.resource.Spec.Hostnames) > 1 { + return fmt.Errorf("too many hostnames in the konnectivity route") + } + + logger.V(1).Info("updating TenantControlPlane status for Konnectivity Gateway routes") + accessPoints, err := resources.BuildGatewayAccessPointsStatus(ctx, r.Client, r.resource, routeStatuses) + if err != nil { + return err + } + tcp.Status.Addons.Konnectivity.Gateway.AccessPoints = accessPoints + + return nil +} + +func (r *KubernetesKonnectivityGatewayResource) Define(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &gatewayv1alpha2.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-konnectivity", tcp.GetName()), + Namespace: tcp.GetNamespace(), + }, + } + + return nil +} + +func (r *KubernetesKonnectivityGatewayResource) mutate(tcp *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + // Use control plane gateway configuration + if tcp.Spec.ControlPlane.Gateway == nil { + return fmt.Errorf("control plane gateway is not configured") + } + + labels := utilities.MergeMaps( + r.resource.GetLabels(), + utilities.KamajiLabels(tcp.GetName(), r.GetName()), + tcp.Spec.ControlPlane.Gateway.AdditionalMetadata.Labels, + ) + r.resource.SetLabels(labels) + + annotations := utilities.MergeMaps( + r.resource.GetAnnotations(), + tcp.Spec.ControlPlane.Gateway.AdditionalMetadata.Annotations, + ) + r.resource.SetAnnotations(annotations) + + // Use hostname from control plane gateway + if len(tcp.Spec.ControlPlane.Gateway.Hostname) == 0 { + return fmt.Errorf("control plane gateway hostname is not set") + } + + serviceName := gatewayv1alpha2.ObjectName(tcp.Status.Addons.Konnectivity.Service.Name) + servicePort := tcp.Status.Addons.Konnectivity.Service.Port + + if serviceName == "" || servicePort == 0 { + return fmt.Errorf("konnectivity service not ready, cannot create TLSRoute") + } + + // Copy parentRefs from control plane gateway and explicitly set port and sectionName fields + if tcp.Spec.ControlPlane.Gateway.GatewayParentRefs == nil { + return fmt.Errorf("control plane gateway parentRefs are not specified") + } + r.resource.Spec.ParentRefs = resources.NewParentRefsSpecWithPortAndSection(tcp.Spec.ControlPlane.Gateway.GatewayParentRefs, servicePort, "konnectivity-server") + + rule := gatewayv1alpha2.TLSRouteRule{ + BackendRefs: []gatewayv1alpha2.BackendRef{ + { + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: serviceName, + Port: &servicePort, + }, + }, + }, + } + + r.resource.Spec.Hostnames = []gatewayv1.Hostname{tcp.Spec.ControlPlane.Gateway.Hostname} + r.resource.Spec.Rules = []gatewayv1alpha2.TLSRouteRule{rule} + + return controllerutil.SetControllerReference(tcp, r.resource, r.Client.Scheme()) + } +} + +func (r *KubernetesKonnectivityGatewayResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + logger := log.FromContext(ctx, "resource", r.GetName()) + + if !r.shouldHaveGateway(tenantControlPlane) { + return controllerutil.OperationResultNone, nil + } + + if tenantControlPlane.Spec.ControlPlane.Gateway == nil { + return controllerutil.OperationResultNone, nil + } + + if len(tenantControlPlane.Spec.ControlPlane.Gateway.Hostname) == 0 { + return controllerutil.OperationResultNone, fmt.Errorf("missing hostname to expose Konnectivity using a Gateway resource") + } + + logger.V(1).Info("creating or updating resource konnectivity gateway routes") + + result, err := utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(tenantControlPlane)) + if err != nil { + return result, err + } + + return result, nil +} + +func (r *KubernetesKonnectivityGatewayResource) GetName() string { + return "konnectivity_gateway_routes" +} diff --git a/internal/resources/konnectivity/gateway_resource_test.go b/internal/resources/konnectivity/gateway_resource_test.go new file mode 100644 index 0000000..0c6c1dd --- /dev/null +++ b/internal/resources/konnectivity/gateway_resource_test.go @@ -0,0 +1,218 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package konnectivity_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/resources/konnectivity" +) + +func TestKonnectivityGatewayResource(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Konnectivity Gateway Resource Suite") +} + +var runtimeScheme *runtime.Scheme + +var _ = BeforeSuite(func() { + runtimeScheme = runtime.NewScheme() + Expect(scheme.AddToScheme(runtimeScheme)).To(Succeed()) + Expect(kamajiv1alpha1.AddToScheme(runtimeScheme)).To(Succeed()) + Expect(gatewayv1alpha2.Install(runtimeScheme)).To(Succeed()) +}) + +var _ = Describe("KubernetesKonnectivityGatewayResource", func() { + var ( + tcp *kamajiv1alpha1.TenantControlPlane + resource *konnectivity.KubernetesKonnectivityGatewayResource + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + + fakeClient := fake.NewClientBuilder(). + WithScheme(runtimeScheme). + Build() + + resource = &konnectivity.KubernetesKonnectivityGatewayResource{ + Client: fakeClient, + } + + namespace := gatewayv1.Namespace("default") + tcp = &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-tcp", + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{ + ControlPlane: kamajiv1alpha1.ControlPlane{ + Gateway: &kamajiv1alpha1.GatewaySpec{ + Hostname: gatewayv1alpha2.Hostname("test.example.com"), + GatewayParentRefs: []gatewayv1alpha2.ParentReference{ + { + Name: "test-gateway", + Namespace: &namespace, + }, + }, + }, + }, + Addons: kamajiv1alpha1.AddonsSpec{ + Konnectivity: &kamajiv1alpha1.KonnectivitySpec{ + KonnectivityServerSpec: kamajiv1alpha1.KonnectivityServerSpec{ + Port: 8132, + }, + }, + }, + }, + Status: kamajiv1alpha1.TenantControlPlaneStatus{ + Addons: kamajiv1alpha1.AddonsStatus{ + Konnectivity: kamajiv1alpha1.KonnectivityStatus{ + Service: kamajiv1alpha1.KubernetesServiceStatus{ + Name: "test-konnectivity-service", + Port: 8132, + }, + }, + }, + }, + } + }) + + Describe("shouldHaveGateway logic", func() { + It("should return false when Konnectivity addon is disabled", func() { + tcp.Spec.Addons.Konnectivity = nil + shouldUpdate := resource.ShouldStatusBeUpdated(ctx, tcp) + Expect(shouldUpdate).To(BeFalse()) + Expect(resource.ShouldCleanup(tcp)).To(BeFalse()) + }) + + It("should return false when control plane gateway is not configured", func() { + tcp.Spec.ControlPlane.Gateway = nil + shouldUpdate := resource.ShouldStatusBeUpdated(ctx, tcp) + Expect(shouldUpdate).To(BeFalse()) + Expect(resource.ShouldCleanup(tcp)).To(BeFalse()) + }) + + It("should return true when both Konnectivity and gateway are configured", func() { + shouldUpdate := resource.ShouldStatusBeUpdated(ctx, tcp) + Expect(shouldUpdate).To(BeTrue()) + Expect(resource.ShouldCleanup(tcp)).To(BeFalse()) + }) + }) + + Context("When Konnectivity gateway should be configured", func() { + It("should set correct TLSRoute name with -konnectivity suffix", func() { + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + _, err = resource.CreateOrUpdate(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + route := &gatewayv1alpha2.TLSRoute{} + err = resource.Client.Get(ctx, client.ObjectKey{Name: "test-tcp-konnectivity", Namespace: tcp.Namespace}, route) + Expect(err).NotTo(HaveOccurred()) + Expect(route.Name).To(Equal("test-tcp-konnectivity")) + }) + + It("should set sectionName to \"konnectivity-server\" and port from Konnectivity service status", func() { + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + _, err = resource.CreateOrUpdate(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + route := &gatewayv1alpha2.TLSRoute{} + err = resource.Client.Get(ctx, client.ObjectKey{Name: "test-tcp-konnectivity", Namespace: tcp.Namespace}, route) + Expect(err).NotTo(HaveOccurred()) + Expect(route.Spec.ParentRefs).To(HaveLen(1)) + Expect(route.Spec.ParentRefs[0].SectionName).NotTo(BeNil()) + Expect(*route.Spec.ParentRefs[0].SectionName).To(Equal(gatewayv1.SectionName("konnectivity-server"))) + Expect(route.Spec.ParentRefs[0].Port).NotTo(BeNil()) + Expect(*route.Spec.ParentRefs[0].Port).To(Equal(tcp.Status.Addons.Konnectivity.Service.Port)) + }) + + It("should use control plane gateway hostname", func() { + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + _, err = resource.CreateOrUpdate(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + route := &gatewayv1alpha2.TLSRoute{} + err = resource.Client.Get(ctx, client.ObjectKey{Name: "test-tcp-konnectivity", Namespace: tcp.Namespace}, route) + Expect(err).NotTo(HaveOccurred()) + Expect(route.Spec.Hostnames).To(HaveLen(1)) + Expect(route.Spec.Hostnames[0]).To(Equal(tcp.Spec.ControlPlane.Gateway.Hostname)) + }) + }) + + Context("Konnectivity-specific error cases", func() { + It("should return early without error when control plane gateway is not configured", func() { + tcp.Spec.ControlPlane.Gateway = nil + + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + result, err := resource.CreateOrUpdate(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(controllerutil.OperationResultNone)) + }) + + It("should fail when Konnectivity service is not ready", func() { + tcp.Status.Addons.Konnectivity.Service.Name = "" + tcp.Status.Addons.Konnectivity.Service.Port = 0 + + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + _, err = resource.CreateOrUpdate(ctx, tcp) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("konnectivity service not ready")) + }) + + It("should fail when control plane gateway parentRefs are not specified", func() { + tcp.Spec.ControlPlane.Gateway.GatewayParentRefs = nil + + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + _, err = resource.CreateOrUpdate(ctx, tcp) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("control plane gateway parentRefs are not specified")) + }) + }) + + Context("When Konnectivity gateway should not be configured", func() { + BeforeEach(func() { + tcp.Spec.Addons.Konnectivity = nil + tcp.Status.Addons.Konnectivity = kamajiv1alpha1.KonnectivityStatus{ + Gateway: &kamajiv1alpha1.KubernetesGatewayStatus{ + AccessPoints: nil, + }, + } + }) + + It("should cleanup when gateway is removed", func() { + Expect(resource.ShouldCleanup(tcp)).To(BeTrue()) + }) + }) + + It("should return correct resource name", func() { + Expect(resource.GetName()).To(Equal("konnectivity_gateway_routes")) + }) +}) diff --git a/internal/resources/konnectivity/metrics.go b/internal/resources/konnectivity/metrics.go index 3bab93b..95ab0de 100644 --- a/internal/resources/konnectivity/metrics.go +++ b/internal/resources/konnectivity/metrics.go @@ -13,6 +13,7 @@ var ( clusterrolebindingCollector prometheus.Histogram deploymentCollector prometheus.Histogram egressCollector prometheus.Histogram + gatewayCollector prometheus.Histogram kubeconfigCollector prometheus.Histogram serviceaccountCollector prometheus.Histogram serviceCollector prometheus.Histogram