mirror of
https://github.com/clastix/kamaji.git
synced 2026-04-06 10:47:37 +00:00
Compare commits
4 Commits
26.3.5-edg
...
26.4.2-edg
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e51909fae3 | ||
|
|
a62592587e | ||
|
|
0578e99dbc | ||
|
|
e27202e0e0 |
2
.github/workflows/helm.yaml
vendored
2
.github/workflows/helm.yaml
vendored
@@ -24,7 +24,7 @@ jobs:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: azure/setup-helm@v4
|
||||
- uses: azure/setup-helm@v5
|
||||
with:
|
||||
version: 3.3.4
|
||||
- name: Building dependencies
|
||||
|
||||
@@ -37,6 +37,22 @@ func (in *TenantControlPlane) AssignedControlPlaneAddress() (string, int32, erro
|
||||
return address, int32(port), nil
|
||||
}
|
||||
|
||||
// AdvertisedControlPlaneAddress returns the address and port to advertise to tenant-side consumers.
|
||||
// If AdvertiseAddress is set, it is returned with the same port as the management address.
|
||||
// Otherwise, it falls back to AssignedControlPlaneAddress.
|
||||
func (in *TenantControlPlane) AdvertisedControlPlaneAddress() (string, int32, error) {
|
||||
if in.Spec.NetworkProfile.AdvertiseAddress != "" {
|
||||
_, port, err := in.AssignedControlPlaneAddress()
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
return in.Spec.NetworkProfile.AdvertiseAddress, port, nil
|
||||
}
|
||||
|
||||
return in.AssignedControlPlaneAddress()
|
||||
}
|
||||
|
||||
// DeclaredControlPlaneAddress returns the desired Tenant Control Plane address.
|
||||
// In case of dynamic allocation, e.g. using a Load Balancer, it queries the API Server looking for the allocated IP.
|
||||
// When an IP has not been yet assigned, or it is expected, an error is returned.
|
||||
|
||||
@@ -32,6 +32,11 @@ type NetworkProfileSpec struct {
|
||||
// Address where API server will be exposed.
|
||||
// In the case of LoadBalancer Service, this can be empty in order to use the exposed IP provided by the cloud controller manager.
|
||||
Address string `json:"address,omitempty"`
|
||||
// AdvertiseAddress is the address advertised to tenant-side consumers (workers, konnectivity).
|
||||
// When set, the management address is used for CAPI and status reporting, while this address
|
||||
// is used for kubeadm ControlPlaneEndpoint, cluster-info, and admin.conf.
|
||||
// Both addresses are included in the API server certificate SANs.
|
||||
AdvertiseAddress string `json:"advertiseAddress,omitempty"`
|
||||
// The default domain name used for DNS resolution within the cluster.
|
||||
//+kubebuilder:default="cluster.local"
|
||||
//+kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the cluster domain is not supported"
|
||||
|
||||
@@ -7579,6 +7579,13 @@ versions:
|
||||
Address where API server will be exposed.
|
||||
In the case of LoadBalancer Service, this can be empty in order to use the exposed IP provided by the cloud controller manager.
|
||||
type: string
|
||||
advertiseAddress:
|
||||
description: |-
|
||||
AdvertiseAddress is the address advertised to tenant-side consumers (workers, konnectivity).
|
||||
When set, the management address is used for CAPI and status reporting, while this address
|
||||
is used for kubeadm ControlPlaneEndpoint, cluster-info, and admin.conf.
|
||||
Both addresses are included in the API server certificate SANs.
|
||||
type: string
|
||||
allowAddressAsExternalIP:
|
||||
description: |-
|
||||
AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address in the section of
|
||||
|
||||
@@ -7587,6 +7587,13 @@ spec:
|
||||
Address where API server will be exposed.
|
||||
In the case of LoadBalancer Service, this can be empty in order to use the exposed IP provided by the cloud controller manager.
|
||||
type: string
|
||||
advertiseAddress:
|
||||
description: |-
|
||||
AdvertiseAddress is the address advertised to tenant-side consumers (workers, konnectivity).
|
||||
When set, the management address is used for CAPI and status reporting, while this address
|
||||
is used for kubeadm ControlPlaneEndpoint, cluster-info, and admin.conf.
|
||||
Both addresses are included in the API server certificate SANs.
|
||||
type: string
|
||||
allowAddressAsExternalIP:
|
||||
description: |-
|
||||
AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address in the section of
|
||||
|
||||
851
docs/content/cluster-api/openstack-infra-provider.md
Normal file
851
docs/content/cluster-api/openstack-infra-provider.md
Normal file
@@ -0,0 +1,851 @@
|
||||
# OpenStack Infra Provider
|
||||
|
||||
Use the Cluster API [OpenStack Infra Provider (CAPO)](https://github.com/kubernetes-sigs/cluster-api-provider-openstack) with the Cluster API [Kamaji Control Plane Provider](https://github.com/clastix/cluster-api-control-plane-provider-kamaji) to create Kubernetes clusters.
|
||||
|
||||
!!! warning "Important Notes"
|
||||
This walkthrough uses an advanced `externalClusterReference` setup. `kind` cluster is used as the Cluster API management cluster for ease of PoC, and it assumes you start with no existing Kubernetes infrastructure, only a working OpenStack environment.
|
||||
|
||||
Because a local `kind` cluster is typically not reachable from OpenStack instances, Kamaji is installed on a kubeadm-based Kubernetes cluster running inside OpenStack (the "control plane cluster"). The tenant control plane is then deployed there using `KamajiControlPlane.spec.deployment.externalClusterReference`.
|
||||
|
||||
If you already have a Kubernetes cluster reachable from your OpenStack nodes (for example, a Magnum-bootstrapped cluster, a kubeadm cluster in OpenStack, or any managed Kubernetes service), you can run both Cluster API controllers and Kamaji on that single management cluster and avoid `externalClusterReference`.
|
||||
|
||||
## Topology
|
||||
|
||||
```text
|
||||
+--------------------+
|
||||
| management cluster |
|
||||
+---------+----------+
|
||||
/ \
|
||||
/ \
|
||||
v v
|
||||
+-----------------------+ +-------------------------+
|
||||
| control plane cluster | | workload tenant cluster |
|
||||
+-----------+-----------+ +------------+------------+
|
||||
\ /
|
||||
+--------------------------+
|
||||
```
|
||||
|
||||
- The management cluster is a local `kind` cluster and runs Cluster API core, Kubeadm providers, CAPO, CAPH (Cluster API Helm add-on provider), and the Kamaji Cluster API control plane provider, and it reconciles both OpenStack `Cluster` resources.
|
||||
- The control plane cluster is a kubeadm-based cluster on OpenStack that runs Kamaji and add-ons such as CCM, CNI, and CSI.
|
||||
- The workload tenant cluster runs worker nodes on OpenStack, while its tenant control plane is hosted by Kamaji on the control plane cluster.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- `kind`, `kubectl`, `clusterctl`, `openstack` CLI, `openssl`, and `base64`
|
||||
- Host images built with [image-builder](https://image-builder.sigs.k8s.io/) (or equivalent) and configured with `cloud-init` using the [OpenStack datasource](https://docs.cloud-init.io/en/latest/reference/datasources/openstack.html)
|
||||
- An existing OpenStack environment with network connectivity, flavors, host images, and sufficient quotas for control plane and worker nodes
|
||||
- OpenStack load balancer support (Octavia) for Service type `LoadBalancer` and API endpoint/public IP management for the kubeadm-based control plane cluster
|
||||
|
||||
!!! warning "ProviderID and cloud-init behavior"
|
||||
CAPI host images must have the OpenStack cloud-init datasource configured. Otherwise, kubelet `provider-id` injection can resolve to a Nova-style non-UUID value and Cluster API reconciliation can stall. In this configuration, `kubeletExtraArgs` does not set `provider-id`; OpenStack CCM sets `providerID` after cluster start. See [cloud-init OpenStack datasource docs](https://docs.cloud-init.io/en/latest/reference/datasources/openstack.html) and [CAPO external cloud provider notes](https://cluster-api-openstack.sigs.k8s.io/topics/external-cloud-provider).
|
||||
|
||||
## Setup management cluster
|
||||
|
||||
```bash
|
||||
kind create cluster --name management-cluster
|
||||
kubectl cluster-info --context kind-management-cluster
|
||||
```
|
||||
|
||||
### Install providers
|
||||
|
||||
Install ORC before initializing Cluster API providers:
|
||||
|
||||
```bash
|
||||
export ORC_VERSION=v2.0.3
|
||||
kubectl apply -f "https://github.com/k-orc/openstack-resource-controller/releases/download/${ORC_VERSION}/install.yaml"
|
||||
```
|
||||
|
||||
Initialize Cluster API providers:
|
||||
|
||||
```bash
|
||||
clusterctl init \
|
||||
--core cluster-api \
|
||||
--bootstrap kubeadm \
|
||||
--control-plane kubeadm \
|
||||
--infrastructure openstack \
|
||||
--addon helm \
|
||||
--control-plane kamaji
|
||||
```
|
||||
|
||||
Enable feature gates required by `KamajiControlPlane.spec.deployment.externalClusterReference`:
|
||||
|
||||
```bash
|
||||
kubectl -n kamaji-system patch deployment capi-kamaji-controller-manager \
|
||||
--type='json' \
|
||||
-p='[
|
||||
{
|
||||
"op": "replace",
|
||||
"path": "/spec/template/spec/containers/0/args/1",
|
||||
"value": "--feature-gates=ExternalClusterReference=true,ExternalClusterReferenceCrossNamespace=true"
|
||||
}
|
||||
]'
|
||||
```
|
||||
|
||||
### Prepare OpenStack application credentials
|
||||
|
||||
Application credentials are used instead of username/password.
|
||||
|
||||
!!! warning "Credential Scope"
|
||||
The same application credential is reused for both tenant clusters. In production, separate credentials per cluster or environment are preferred.
|
||||
|
||||
!!! note "Project selection"
|
||||
Application credentials are created in the currently scoped project. Set `TARGET_PROJECT_ID` explicitly so credential creation is project-scoped.
|
||||
|
||||
Set and review variables:
|
||||
|
||||
```bash
|
||||
export OPENSTACK_CLOUD_NAME=example-openstack
|
||||
export TARGET_PROJECT_ID=<REPLACE_WITH_PROJECT_ID>
|
||||
export OPENSTACK_APP_CREDENTIAL_NAME=capi-kamaji
|
||||
export OPENSTACK_APP_CREDENTIAL_SECRET="$(openssl rand -hex 24)"
|
||||
```
|
||||
|
||||
Create the application credential:
|
||||
|
||||
```bash
|
||||
openstack --os-cloud "$OPENSTACK_CLOUD_NAME" application credential create \
|
||||
--os-project-id "$TARGET_PROJECT_ID" \
|
||||
--secret "$OPENSTACK_APP_CREDENTIAL_SECRET" \
|
||||
"$OPENSTACK_APP_CREDENTIAL_NAME"
|
||||
```
|
||||
|
||||
Store the generated credential ID:
|
||||
|
||||
```bash
|
||||
export OPENSTACK_APP_CREDENTIAL_ID="$(openstack --os-cloud "$OPENSTACK_CLOUD_NAME" --os-project-id "$TARGET_PROJECT_ID" application credential show "$OPENSTACK_APP_CREDENTIAL_NAME" -f value -c id)"
|
||||
```
|
||||
|
||||
### Prepare `clouds.yaml` and `cloud.conf`
|
||||
|
||||
Set OpenStack connection variables:
|
||||
|
||||
```bash
|
||||
export OPENSTACK_AUTH_URL=https://openstack.example.com:5000/v3
|
||||
export OPENSTACK_REGION_NAME=RegionOne
|
||||
export OPENSTACK_INTERFACE=public
|
||||
export OPENSTACK_IDENTITY_API_VERSION=3
|
||||
export OPENSTACK_TLS_INSECURE=false
|
||||
```
|
||||
|
||||
Build `clouds.yaml` and `cloud.conf` content in environment variables:
|
||||
|
||||
```bash
|
||||
export CLOUDS_YAML_CONTENT="$(cat <<EOF
|
||||
clouds:
|
||||
${OPENSTACK_CLOUD_NAME}:
|
||||
auth:
|
||||
auth_url: ${OPENSTACK_AUTH_URL}
|
||||
application_credential_id: ${OPENSTACK_APP_CREDENTIAL_ID}
|
||||
application_credential_secret: ${OPENSTACK_APP_CREDENTIAL_SECRET}
|
||||
auth_type: v3applicationcredential
|
||||
region_name: ${OPENSTACK_REGION_NAME}
|
||||
interface: ${OPENSTACK_INTERFACE}
|
||||
identity_api_version: ${OPENSTACK_IDENTITY_API_VERSION}
|
||||
EOF
|
||||
)"
|
||||
|
||||
export CLOUD_CONF_CONTENT="$(cat <<EOF
|
||||
[Global]
|
||||
auth-url=${OPENSTACK_AUTH_URL}
|
||||
application-credential-id=${OPENSTACK_APP_CREDENTIAL_ID}
|
||||
application-credential-secret=${OPENSTACK_APP_CREDENTIAL_SECRET}
|
||||
region=${OPENSTACK_REGION_NAME}
|
||||
tls-insecure=${OPENSTACK_TLS_INSECURE}
|
||||
interface=public
|
||||
identity-api-version=3
|
||||
auth-type=v3applicationcredential
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
Encode both values for Kubernetes Secret `data` fields:
|
||||
|
||||
```bash
|
||||
export REPLACE_BASE64_CLOUD_CONF="$(printf '%s' "$CLOUD_CONF_CONTENT" | base64 | tr -d '\n')"
|
||||
export REPLACE_BASE64_CLOUDS_YAML="$(printf '%s' "$CLOUDS_YAML_CONTENT" | base64 | tr -d '\n')"
|
||||
```
|
||||
|
||||
## Bootstrap the control plane cluster
|
||||
|
||||
### Set variables
|
||||
|
||||
Set shared variables used by the next commands:
|
||||
|
||||
!!! note "Variable Values"
|
||||
Names, versions, network IDs, flavors, and image names in this section are example values. Replace them with environment-specific OpenStack values.
|
||||
|
||||
!!! info "Namespace Layout"
|
||||
All resources are created in `kamaji-system`. Resources can also be separated by namespace with corresponding RBAC and policy controls.
|
||||
|
||||
```bash
|
||||
export CLUSTER_NAMESPACE=kamaji-system
|
||||
export CONTROL_PLANE_CLUSTER_NAME=kamaji-control-plane
|
||||
export WORKLOAD_TENANT_CLUSTER_NAME=tenant-cluster-01
|
||||
export KUBERNETES_VERSION=v1.33.0
|
||||
export OPENSTACK_EXTERNAL_NETWORK_ID=00000000-0000-0000-0000-000000000000
|
||||
export OPENSTACK_FLAVOR=m1.large
|
||||
export OPENSTACK_IMAGE_NAME=ubuntu-2404-kube-v1.33.0
|
||||
export OPENSTACK_SSH_KEY_NAME=default
|
||||
```
|
||||
|
||||
### Apply cluster resources
|
||||
|
||||
```bash
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
labels:
|
||||
clusterctl.cluster.x-k8s.io/move: "true"
|
||||
type: Opaque
|
||||
data:
|
||||
cacert: ""
|
||||
clouds.yaml: ${REPLACE_BASE64_CLOUDS_YAML}
|
||||
---
|
||||
apiVersion: cluster.x-k8s.io/v1beta1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
labels:
|
||||
addons.cluster.x-k8s.io/ccm: "true"
|
||||
addons.cluster.x-k8s.io/cilium: "true"
|
||||
addons.cluster.x-k8s.io/cinder-csi: "true"
|
||||
addons.cluster.x-k8s.io/cert-manager: "true"
|
||||
addons.cluster.x-k8s.io/kamaji: "true"
|
||||
spec:
|
||||
clusterNetwork:
|
||||
services:
|
||||
cidrBlocks:
|
||||
- 10.96.0.0/12
|
||||
pods:
|
||||
cidrBlocks:
|
||||
- 10.244.0.0/16
|
||||
serviceDomain: cluster.local
|
||||
controlPlaneRef:
|
||||
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
|
||||
kind: KubeadmControlPlane
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}-control-plane
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
infrastructureRef:
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
|
||||
kind: OpenStackCluster
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
---
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
|
||||
kind: OpenStackCluster
|
||||
metadata:
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
apiServerLoadBalancer:
|
||||
enabled: true
|
||||
externalNetwork:
|
||||
id: ${OPENSTACK_EXTERNAL_NETWORK_ID}
|
||||
identityRef:
|
||||
cloudName: ${OPENSTACK_CLOUD_NAME}
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}
|
||||
managedSecurityGroups:
|
||||
allowAllInClusterTraffic: true
|
||||
managedSubnets:
|
||||
- cidr: 10.0.0.0/24
|
||||
dnsNameservers:
|
||||
- 8.8.8.8
|
||||
---
|
||||
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
|
||||
kind: KubeadmControlPlane
|
||||
metadata:
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}-control-plane
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
replicas: 1
|
||||
version: ${KUBERNETES_VERSION}
|
||||
machineTemplate:
|
||||
infrastructureRef:
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
|
||||
kind: OpenStackMachineTemplate
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}-control-plane
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
kubeadmConfigSpec:
|
||||
format: cloud-config
|
||||
files:
|
||||
- path: /etc/kubernetes/cloud.conf
|
||||
owner: root:root
|
||||
permissions: "0600"
|
||||
encoding: base64
|
||||
content: ${REPLACE_BASE64_CLOUD_CONF}
|
||||
clusterConfiguration:
|
||||
apiServer:
|
||||
extraArgs:
|
||||
cloud-provider: external
|
||||
controllerManager:
|
||||
extraArgs:
|
||||
cloud-provider: external
|
||||
initConfiguration:
|
||||
nodeRegistration:
|
||||
kubeletExtraArgs:
|
||||
cloud-provider: external
|
||||
name: '{{ local_hostname }}'
|
||||
joinConfiguration:
|
||||
nodeRegistration:
|
||||
kubeletExtraArgs:
|
||||
cloud-provider: external
|
||||
name: '{{ local_hostname }}'
|
||||
---
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
|
||||
kind: OpenStackMachineTemplate
|
||||
metadata:
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}-control-plane
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
flavor: ${OPENSTACK_FLAVOR}
|
||||
image:
|
||||
filter:
|
||||
name: ${OPENSTACK_IMAGE_NAME}
|
||||
sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
|
||||
---
|
||||
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
|
||||
kind: KubeadmConfigTemplate
|
||||
metadata:
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}-bootstrap
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
format: cloud-config
|
||||
files:
|
||||
- path: /etc/kubernetes/cloud.conf
|
||||
owner: root:root
|
||||
permissions: "0600"
|
||||
encoding: base64
|
||||
content: ${REPLACE_BASE64_CLOUD_CONF}
|
||||
joinConfiguration:
|
||||
nodeRegistration:
|
||||
imagePullPolicy: IfNotPresent
|
||||
kubeletExtraArgs:
|
||||
cloud-provider: external
|
||||
name: '{{ local_hostname }}'
|
||||
---
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
|
||||
kind: OpenStackMachineTemplate
|
||||
metadata:
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}-worker
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
flavor: ${OPENSTACK_FLAVOR}
|
||||
image:
|
||||
filter:
|
||||
name: ${OPENSTACK_IMAGE_NAME}
|
||||
sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
|
||||
---
|
||||
apiVersion: cluster.x-k8s.io/v1beta1
|
||||
kind: MachineDeployment
|
||||
metadata:
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}-md-0
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
clusterName: ${CONTROL_PLANE_CLUSTER_NAME}
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
nodepool: worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
nodepool: worker
|
||||
spec:
|
||||
clusterName: ${CONTROL_PLANE_CLUSTER_NAME}
|
||||
version: ${KUBERNETES_VERSION}
|
||||
bootstrap:
|
||||
configRef:
|
||||
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
|
||||
kind: KubeadmConfigTemplate
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}-bootstrap
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
infrastructureRef:
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
|
||||
kind: OpenStackMachineTemplate
|
||||
name: ${CONTROL_PLANE_CLUSTER_NAME}-worker
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
EOF
|
||||
```
|
||||
|
||||
### Apply add-on Helm release resources
|
||||
|
||||
!!! info "Add-on scheduling profile"
|
||||
HelmChartProxy resources are configured to match both the control plane cluster and the workload tenant cluster. The chart values are set to work on both kubeadm-based and Kamaji-based clusters (for example, `nodeSelector: null`, broad tolerations, and `dnsPolicy: Default`).
|
||||
|
||||
```bash
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: addons.cluster.x-k8s.io/v1alpha1
|
||||
kind: HelmChartProxy
|
||||
metadata:
|
||||
name: openstack-cloud-controller-manager
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
clusterSelector:
|
||||
matchLabels:
|
||||
addons.cluster.x-k8s.io/ccm: "true"
|
||||
repoURL: https://kubernetes.github.io/cloud-provider-openstack
|
||||
chartName: openstack-cloud-controller-manager
|
||||
version: 2.30.1
|
||||
namespace: kube-system
|
||||
releaseName: openstack-cloud-controller-manager
|
||||
options:
|
||||
enableClientCache: false
|
||||
timeout: 10m0s
|
||||
install:
|
||||
createNamespace: true
|
||||
upgrade:
|
||||
maxHistory: 10
|
||||
valuesTemplate: |
|
||||
secret:
|
||||
enabled: false
|
||||
create: false
|
||||
|
||||
nodeSelector: null
|
||||
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cloudprovider.kubernetes.io/uninitialized
|
||||
operator: Equal
|
||||
value: "true"
|
||||
effect: NoSchedule
|
||||
- key: node.kubernetes.io/not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cluster.x-k8s.io/uninitialized
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
podSecurityContext:
|
||||
runAsUser: 0
|
||||
|
||||
dnsPolicy: Default
|
||||
|
||||
extraVolumes:
|
||||
- name: cloud-config
|
||||
hostPath:
|
||||
path: /etc/kubernetes/cloud.conf
|
||||
type: File
|
||||
- name: k8s-certs
|
||||
hostPath:
|
||||
path: /etc/kubernetes/pki
|
||||
|
||||
extraVolumeMounts:
|
||||
- name: cloud-config
|
||||
mountPath: /etc/config/cloud.conf
|
||||
readOnly: true
|
||||
- name: k8s-certs
|
||||
mountPath: /etc/kubernetes/pki
|
||||
readOnly: true
|
||||
---
|
||||
apiVersion: addons.cluster.x-k8s.io/v1alpha1
|
||||
kind: HelmChartProxy
|
||||
metadata:
|
||||
name: cilium
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
clusterSelector:
|
||||
matchLabels:
|
||||
addons.cluster.x-k8s.io/cilium: "true"
|
||||
repoURL: https://helm.cilium.io
|
||||
chartName: cilium
|
||||
version: 1.18.4
|
||||
namespace: kube-system
|
||||
releaseName: cilium
|
||||
options:
|
||||
enableClientCache: false
|
||||
timeout: 10m0s
|
||||
install:
|
||||
createNamespace: true
|
||||
upgrade:
|
||||
maxHistory: 10
|
||||
valuesTemplate: |
|
||||
cni:
|
||||
chainingMode: portmap
|
||||
|
||||
prometheus:
|
||||
enabled: false
|
||||
|
||||
operator:
|
||||
replicas: 1
|
||||
prometheus:
|
||||
enabled: false
|
||||
|
||||
ipam:
|
||||
operator:
|
||||
clusterPoolIPv4PodCIDRList:
|
||||
- "10.244.0.0/16"
|
||||
clusterPoolIPv4MaskSize: 24
|
||||
|
||||
kubeProxyReplacement: false
|
||||
sessionAffinity: true
|
||||
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cloudprovider.kubernetes.io/uninitialized
|
||||
operator: Equal
|
||||
value: "true"
|
||||
effect: NoSchedule
|
||||
- key: node.kubernetes.io/not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
---
|
||||
apiVersion: addons.cluster.x-k8s.io/v1alpha1
|
||||
kind: HelmChartProxy
|
||||
metadata:
|
||||
name: openstack-cinder-csi
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
clusterSelector:
|
||||
matchLabels:
|
||||
addons.cluster.x-k8s.io/cinder-csi: "true"
|
||||
repoURL: https://kubernetes.github.io/cloud-provider-openstack
|
||||
chartName: openstack-cinder-csi
|
||||
version: 2.31.7
|
||||
namespace: kube-system
|
||||
releaseName: openstack-cinder-csi
|
||||
options:
|
||||
enableClientCache: false
|
||||
timeout: 10m0s
|
||||
install:
|
||||
createNamespace: true
|
||||
upgrade:
|
||||
maxHistory: 10
|
||||
valuesTemplate: |
|
||||
csi:
|
||||
plugin:
|
||||
nodePlugin:
|
||||
dnsPolicy: Default
|
||||
controllerPlugin:
|
||||
dnsPolicy: Default
|
||||
nodeSelector: null
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cloudprovider.kubernetes.io/uninitialized
|
||||
operator: Equal
|
||||
value: "true"
|
||||
effect: NoSchedule
|
||||
- key: node.kubernetes.io/not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
- key: node.cluster.x-k8s.io/uninitialized
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
secret:
|
||||
enabled: false
|
||||
create: false
|
||||
hostMount: true
|
||||
filename: cloud.conf
|
||||
|
||||
storageClass:
|
||||
enabled: false
|
||||
custom: |-
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
annotations:
|
||||
storageclass.kubernetes.io/is-default-class: "true"
|
||||
labels:
|
||||
name: cinder-ssd
|
||||
name: cinder-ssd
|
||||
allowVolumeExpansion: true
|
||||
provisioner: cinder.csi.openstack.org
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: Immediate
|
||||
parameters:
|
||||
type: ssd
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
labels:
|
||||
name: cinder-hdd
|
||||
name: cinder-hdd
|
||||
allowVolumeExpansion: true
|
||||
provisioner: cinder.csi.openstack.org
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: Immediate
|
||||
parameters:
|
||||
type: hdd
|
||||
---
|
||||
apiVersion: addons.cluster.x-k8s.io/v1alpha1
|
||||
kind: HelmChartProxy
|
||||
metadata:
|
||||
name: cert-manager
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
clusterSelector:
|
||||
matchLabels:
|
||||
addons.cluster.x-k8s.io/cert-manager: "true"
|
||||
repoURL: https://charts.jetstack.io
|
||||
chartName: cert-manager
|
||||
namespace: cert-manager
|
||||
releaseName: cert-manager
|
||||
options:
|
||||
enableClientCache: false
|
||||
timeout: 10m0s
|
||||
install:
|
||||
createNamespace: true
|
||||
upgrade:
|
||||
maxHistory: 10
|
||||
valuesTemplate: |
|
||||
installCRDs: true
|
||||
---
|
||||
apiVersion: addons.cluster.x-k8s.io/v1alpha1
|
||||
kind: HelmChartProxy
|
||||
metadata:
|
||||
name: kamaji
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
clusterSelector:
|
||||
matchLabels:
|
||||
addons.cluster.x-k8s.io/kamaji: "true"
|
||||
repoURL: https://clastix.github.io/charts
|
||||
chartName: kamaji
|
||||
version: 0.0.0+latest
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
releaseName: kamaji
|
||||
options:
|
||||
enableClientCache: false
|
||||
timeout: 10m0s
|
||||
install:
|
||||
createNamespace: true
|
||||
upgrade:
|
||||
maxHistory: 10
|
||||
valuesTemplate: |
|
||||
resources: null
|
||||
EOF
|
||||
```
|
||||
|
||||
### Verify control plane cluster
|
||||
|
||||
Apply and monitor:
|
||||
|
||||
```bash
|
||||
clusterctl describe cluster "$CONTROL_PLANE_CLUSTER_NAME" -n "$CLUSTER_NAMESPACE"
|
||||
clusterctl get kubeconfig "$CONTROL_PLANE_CLUSTER_NAME" -n "$CLUSTER_NAMESPACE" > ~/.kube/${CONTROL_PLANE_CLUSTER_NAME}.kubeconfig
|
||||
KUBECONFIG=~/.kube/${CONTROL_PLANE_CLUSTER_NAME}.kubeconfig kubectl get nodes
|
||||
```
|
||||
|
||||
## Bootstrap the tenant cluster with Kamaji control plane
|
||||
|
||||
### Set cross-cluster reference
|
||||
|
||||
Set the reference to the first cluster kubeconfig Secret:
|
||||
|
||||
```bash
|
||||
export CONTROL_PLANE_CLUSTER_KUBECONFIG_SECRET_NAME="${CONTROL_PLANE_CLUSTER_NAME}-kubeconfig"
|
||||
```
|
||||
|
||||
### Apply tenant cluster resources
|
||||
|
||||
```bash
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
labels:
|
||||
clusterctl.cluster.x-k8s.io/move: "true"
|
||||
type: Opaque
|
||||
data:
|
||||
cacert: ""
|
||||
clouds.yaml: ${REPLACE_BASE64_CLOUDS_YAML}
|
||||
---
|
||||
apiVersion: cluster.x-k8s.io/v1beta1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
labels:
|
||||
addons.cluster.x-k8s.io/ccm: "true"
|
||||
addons.cluster.x-k8s.io/cilium: "true"
|
||||
addons.cluster.x-k8s.io/cinder-csi: "true"
|
||||
spec:
|
||||
clusterNetwork:
|
||||
services:
|
||||
cidrBlocks:
|
||||
- 10.96.0.0/12
|
||||
pods:
|
||||
cidrBlocks:
|
||||
- 10.244.0.0/16
|
||||
serviceDomain: cluster.local
|
||||
controlPlaneRef:
|
||||
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
|
||||
kind: KamajiControlPlane
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
infrastructureRef:
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
|
||||
kind: OpenStackCluster
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
---
|
||||
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
|
||||
kind: KamajiControlPlane
|
||||
metadata:
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
version: ${KUBERNETES_VERSION}
|
||||
dataStoreName: default
|
||||
replicas: 1
|
||||
apiServer:
|
||||
extraArgs:
|
||||
- --cloud-provider=external
|
||||
controllerManager:
|
||||
extraArgs:
|
||||
- --cloud-provider=external
|
||||
kubelet:
|
||||
cgroupfs: systemd
|
||||
preferredAddressTypes:
|
||||
- InternalIP
|
||||
configurationJSONPatches:
|
||||
- op: remove
|
||||
path: /imagePullCredentialsVerificationPolicy
|
||||
- op: remove
|
||||
path: /mergeDefaultEvictionSettings
|
||||
- op: remove
|
||||
path: /crashLoopBackOff
|
||||
- op: add
|
||||
path: /cgroupDriver
|
||||
value: systemd
|
||||
network:
|
||||
serviceType: LoadBalancer
|
||||
addons:
|
||||
coreDNS: {}
|
||||
kubeProxy: {}
|
||||
konnectivity: {}
|
||||
deployment:
|
||||
externalClusterReference:
|
||||
deploymentNamespace: ${CLUSTER_NAMESPACE}
|
||||
kubeconfigSecretName: ${CONTROL_PLANE_CLUSTER_KUBECONFIG_SECRET_NAME}
|
||||
kubeconfigSecretKey: value
|
||||
kubeconfigSecretNamespace: ${CLUSTER_NAMESPACE}
|
||||
---
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
|
||||
kind: OpenStackCluster
|
||||
metadata:
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
apiServerLoadBalancer:
|
||||
enabled: false
|
||||
disableAPIServerFloatingIP: true
|
||||
externalNetwork:
|
||||
id: ${OPENSTACK_EXTERNAL_NETWORK_ID}
|
||||
identityRef:
|
||||
cloudName: ${OPENSTACK_CLOUD_NAME}
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}
|
||||
managedSecurityGroups:
|
||||
allowAllInClusterTraffic: true
|
||||
managedSubnets:
|
||||
- cidr: 10.0.0.0/24
|
||||
dnsNameservers:
|
||||
- 8.8.8.8
|
||||
---
|
||||
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
|
||||
kind: KubeadmConfigTemplate
|
||||
metadata:
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}-bootstrap
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
format: cloud-config
|
||||
files:
|
||||
- path: /etc/kubernetes/cloud.conf
|
||||
owner: root:root
|
||||
permissions: "0600"
|
||||
encoding: base64
|
||||
content: ${REPLACE_BASE64_CLOUD_CONF}
|
||||
joinConfiguration:
|
||||
nodeRegistration:
|
||||
imagePullPolicy: IfNotPresent
|
||||
kubeletExtraArgs:
|
||||
cloud-provider: external
|
||||
name: '{{ local_hostname }}'
|
||||
---
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
|
||||
kind: OpenStackMachineTemplate
|
||||
metadata:
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}-worker
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
flavor: ${OPENSTACK_FLAVOR}
|
||||
image:
|
||||
filter:
|
||||
name: ${OPENSTACK_IMAGE_NAME}
|
||||
sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
|
||||
---
|
||||
apiVersion: cluster.x-k8s.io/v1beta1
|
||||
kind: MachineDeployment
|
||||
metadata:
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}-md-0
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
spec:
|
||||
clusterName: ${WORKLOAD_TENANT_CLUSTER_NAME}
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
nodepool: worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
nodepool: worker
|
||||
spec:
|
||||
clusterName: ${WORKLOAD_TENANT_CLUSTER_NAME}
|
||||
version: ${KUBERNETES_VERSION}
|
||||
bootstrap:
|
||||
configRef:
|
||||
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
|
||||
kind: KubeadmConfigTemplate
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}-bootstrap
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
infrastructureRef:
|
||||
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
|
||||
kind: OpenStackMachineTemplate
|
||||
name: ${WORKLOAD_TENANT_CLUSTER_NAME}-worker
|
||||
namespace: ${CLUSTER_NAMESPACE}
|
||||
EOF
|
||||
```
|
||||
|
||||
### Verify tenant cluster
|
||||
|
||||
Apply and monitor:
|
||||
|
||||
```bash
|
||||
clusterctl describe cluster "$WORKLOAD_TENANT_CLUSTER_NAME" -n "$CLUSTER_NAMESPACE"
|
||||
clusterctl get kubeconfig "$WORKLOAD_TENANT_CLUSTER_NAME" -n "$CLUSTER_NAMESPACE" > ~/.kube/${WORKLOAD_TENANT_CLUSTER_NAME}.kubeconfig
|
||||
KUBECONFIG=~/.kube/${WORKLOAD_TENANT_CLUSTER_NAME}.kubeconfig kubectl get nodes
|
||||
```
|
||||
|
||||
## Clean up
|
||||
|
||||
```bash
|
||||
kubectl delete cluster "$WORKLOAD_TENANT_CLUSTER_NAME" -n "$CLUSTER_NAMESPACE"
|
||||
kubectl delete cluster "$CONTROL_PLANE_CLUSTER_NAME" -n "$CLUSTER_NAMESPACE"
|
||||
kind delete cluster --name management-cluster
|
||||
```
|
||||
@@ -45889,6 +45889,16 @@ NetworkProfile specifies how the network is
|
||||
In the case of LoadBalancer Service, this can be empty in order to use the exposed IP provided by the cloud controller manager.<br/>
|
||||
</td>
|
||||
<td>false</td>
|
||||
</tr><tr>
|
||||
<td><b>advertiseAddress</b></td>
|
||||
<td>string</td>
|
||||
<td>
|
||||
AdvertiseAddress is the address advertised to tenant-side consumers (workers, konnectivity).
|
||||
When set, the management address is used for CAPI and status reporting, while this address
|
||||
is used for kubeadm ControlPlaneEndpoint, cluster-info, and admin.conf.
|
||||
Both addresses are included in the API server certificate SANs.<br/>
|
||||
</td>
|
||||
<td>false</td>
|
||||
</tr><tr>
|
||||
<td><b>allowAddressAsExternalIP</b></td>
|
||||
<td>boolean</td>
|
||||
|
||||
@@ -65,6 +65,7 @@ nav:
|
||||
- cluster-api/control-plane-provider.md
|
||||
- cluster-api/vsphere-infra-provider.md
|
||||
- cluster-api/proxmox-infra-provider.md
|
||||
- cluster-api/openstack-infra-provider.md
|
||||
- cluster-api/other-providers.md
|
||||
- cluster-api/cluster-autoscaler.md
|
||||
- cluster-api/cluster-class.md
|
||||
|
||||
20
go.mod
20
go.mod
@@ -15,7 +15,7 @@ require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/juju/mutex/v2 v2.0.0
|
||||
github.com/nats-io/nats.go v1.49.0
|
||||
github.com/nats-io/nats.go v1.50.0
|
||||
github.com/onsi/ginkgo/v2 v2.28.1
|
||||
github.com/onsi/gomega v1.39.1
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
@@ -99,7 +99,7 @@ require (
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
|
||||
github.com/lithammer/dedent v1.1.0 // indirect
|
||||
@@ -118,7 +118,7 @@ require (
|
||||
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nkeys v0.4.12 // indirect
|
||||
github.com/nats-io/nkeys v0.4.15 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
@@ -164,17 +164,17 @@ require (
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/crypto v0.49.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
|
||||
golang.org/x/mod v0.32.0 // indirect
|
||||
golang.org/x/mod v0.33.0 // indirect
|
||||
golang.org/x/net v0.51.0 // indirect
|
||||
golang.org/x/oauth2 v0.34.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/term v0.40.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/term v0.41.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
golang.org/x/tools v0.41.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
google.golang.org/grpc v1.79.3 // indirect
|
||||
|
||||
40
go.sum
40
go.sum
@@ -192,8 +192,8 @@ github.com/juju/version/v2 v2.0.0-20211007103408-2e8da085dc23/go.mod h1:Ljlbryh9
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
@@ -245,10 +245,10 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
||||
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
|
||||
github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
|
||||
github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
|
||||
github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
|
||||
github.com/nats-io/nats.go v1.50.0 h1:5zAeQrTvyrKrWLJ0fu02W3br8ym57qf7csDzgLOpcds=
|
||||
github.com/nats-io/nats.go v1.50.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
|
||||
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
|
||||
github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
||||
@@ -422,15 +422,15 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
|
||||
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
|
||||
golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
|
||||
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
|
||||
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
@@ -444,8 +444,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -457,15 +457,15 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
|
||||
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
|
||||
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -473,8 +473,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
|
||||
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
@@ -718,10 +718,18 @@ func (d Deployment) buildKubeAPIServerCommand(tenantControlPlane kamajiv1alpha1.
|
||||
kubeletPreferredAddressTypes = append(kubeletPreferredAddressTypes, string(addressType))
|
||||
}
|
||||
|
||||
// Use the advertiseAddress (tenant-facing VIP) for --advertise-address when set.
|
||||
// This ensures the kubernetes endpoint in the tenant cluster points to the VIP,
|
||||
// allowing pods to reach the API server via the OVN localport path.
|
||||
apiAdvertiseAddress := address
|
||||
if adv := tenantControlPlane.Spec.NetworkProfile.AdvertiseAddress; adv != "" {
|
||||
apiAdvertiseAddress = adv
|
||||
}
|
||||
|
||||
desiredArgs := map[string]string{
|
||||
"--allow-privileged": "true",
|
||||
"--authorization-mode": "Node,RBAC",
|
||||
"--advertise-address": address,
|
||||
"--advertise-address": apiAdvertiseAddress,
|
||||
"--client-ca-file": path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName),
|
||||
"--enable-admission-plugins": strings.Join(tenantControlPlane.Spec.Kubernetes.AdmissionControllers.ToSlice(), ","),
|
||||
"--enable-bootstrap-token-auth": "true",
|
||||
|
||||
@@ -175,7 +175,7 @@ func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.T
|
||||
return func() error {
|
||||
logger := log.FromContext(ctx, "resource", r.GetName())
|
||||
|
||||
address, _, err := tenantControlPlane.AssignedControlPlaneAddress()
|
||||
address, _, err := tenantControlPlane.AdvertisedControlPlaneAddress()
|
||||
if err != nil {
|
||||
logger.Error(err, "unable to retrieve the Tenant Control Plane address")
|
||||
|
||||
|
||||
@@ -88,9 +88,15 @@ func (r *KubeadmConfigResource) mutate(ctx context.Context, tenantControlPlane *
|
||||
return err
|
||||
}
|
||||
|
||||
// Resolve the advertised address for tenant-side consumers
|
||||
advAddress, _, advErr := tenantControlPlane.AdvertisedControlPlaneAddress()
|
||||
if advErr != nil {
|
||||
return advErr
|
||||
}
|
||||
|
||||
r.resource.SetLabels(utilities.MergeMaps(r.resource.GetLabels(), utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName())))
|
||||
|
||||
endpoint := net.JoinHostPort(address, strconv.FormatInt(int64(port), 10))
|
||||
endpoint := net.JoinHostPort(advAddress, strconv.FormatInt(int64(port), 10))
|
||||
spec := tenantControlPlane.Spec.ControlPlane
|
||||
if spec.Gateway != nil {
|
||||
if len(spec.Gateway.Hostname) > 0 {
|
||||
@@ -105,13 +111,19 @@ func (r *KubeadmConfigResource) mutate(ctx context.Context, tenantControlPlane *
|
||||
}
|
||||
}
|
||||
|
||||
// Add advertise address to cert SANs if different from management address
|
||||
certSANs := tenantControlPlane.Spec.NetworkProfile.CertSANs
|
||||
if advAddress != address {
|
||||
certSANs = append(append([]string{}, certSANs...), advAddress)
|
||||
}
|
||||
|
||||
params := kubeadm.Parameters{
|
||||
TenantControlPlaneAddress: address,
|
||||
TenantControlPlanePort: port,
|
||||
TenantControlPlaneName: tenantControlPlane.GetName(),
|
||||
TenantControlPlaneNamespace: tenantControlPlane.GetNamespace(),
|
||||
TenantControlPlaneEndpoint: endpoint,
|
||||
TenantControlPlaneCertSANs: tenantControlPlane.Spec.NetworkProfile.CertSANs,
|
||||
TenantControlPlaneCertSANs: certSANs,
|
||||
TenantControlPlaneClusterDomain: tenantControlPlane.Spec.NetworkProfile.ClusterDomain,
|
||||
TenantControlPlanePodCIDR: tenantControlPlane.Spec.NetworkProfile.PodCIDR,
|
||||
TenantControlPlaneServiceCIDR: tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
|
||||
|
||||
Reference in New Issue
Block a user