From 9faab4f82dc635cc073056746065cd75b78e0ad8 Mon Sep 17 00:00:00 2001 From: Michael Bolot Date: Thu, 5 Sep 2024 14:50:11 -0500 Subject: [PATCH] Changing the cluster to be namespaced (#110) * Changing the cluster to be namespaced Changes the cluster type to be namespaced (and changes the various controllers to work with this new feature). Also adds crd generation and docs to the core cluster type. * CI fix --- Dockerfile.dapper | 8 +- charts/k3k/crds/cluster.yaml | 134 -------------- charts/k3k/crds/k3k.io_clusters.yaml | 206 ++++++++++++++++++++++ cli/cmds/cluster/create.go | 6 +- cli/cmds/cluster/delete.go | 3 +- cli/cmds/kubeconfig/kubeconfig.go | 4 +- cli/cmds/root.go | 17 ++ main.go | 4 + ops/build-crds | 8 + pkg/apis/k3k.io/v1alpha1/types.go | 59 +++++-- pkg/controller/cluster/agent/agent.go | 59 ++++--- pkg/controller/cluster/cluster.go | 213 +--------------------- pkg/controller/cluster/config/agent.go | 2 +- pkg/controller/cluster/config/server.go | 12 +- pkg/controller/cluster/pod.go | 214 +++++++++++++++++++++++ pkg/controller/cluster/server/ingress.go | 2 +- pkg/controller/cluster/server/server.go | 42 +++-- pkg/controller/cluster/server/service.go | 2 +- pkg/controller/kubeconfig/kubeconfig.go | 2 +- pkg/controller/util/util.go | 19 +- 20 files changed, 605 insertions(+), 411 deletions(-) delete mode 100644 charts/k3k/crds/cluster.yaml create mode 100644 charts/k3k/crds/k3k.io_clusters.yaml create mode 100755 ops/build-crds create mode 100644 pkg/controller/cluster/pod.go diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 018aefe4..0037d51b 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -6,16 +6,20 @@ ENV ARCH $DAPPER_HOST_ARCH RUN apk -U add \bash git gcc musl-dev docker vim less file curl wget ca-certificates RUN if [ "${ARCH}" == "amd64" ]; then \ - curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.15.0; \ + curl -sL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.59.0; \ fi RUN curl -sL https://github.com/helm/chart-releaser/releases/download/v1.5.0/chart-releaser_1.5.0_linux_${ARCH}.tar.gz | tar -xz cr \ && mv cr /bin/ +# Tool for CRD generation. +ENV CONTROLLER_GEN_VERSION v0.14.0 +RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_GEN_VERSION} + ENV GO111MODULE on ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS GITHUB_TOKEN ENV DAPPER_SOURCE /go/src/github.com/rancher/k3k/ -ENV DAPPER_OUTPUT ./bin ./dist ./deploy +ENV DAPPER_OUTPUT ./bin ./dist ./deploy ./charts ENV DAPPER_DOCKER_SOCKET true ENV HOME ${DAPPER_SOURCE} WORKDIR ${DAPPER_SOURCE} diff --git a/charts/k3k/crds/cluster.yaml b/charts/k3k/crds/cluster.yaml deleted file mode 100644 index d9cc5b41..00000000 --- a/charts/k3k/crds/cluster.yaml +++ /dev/null @@ -1,134 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clusters.k3k.io -spec: - group: k3k.io - versions: - - name: v1alpha1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - name: - type: string - version: - type: string - servers: - type: integer - x-kubernetes-validations: - - message: cluster must have at least one server - rule: self >= 1 - agents: - type: integer - x-kubernetes-validations: - - message: invalid value for agents - rule: self >= 0 - token: - type: string - x-kubernetes-validations: - - message: token is immutable - rule: self == oldSelf - clusterCIDR: - type: string - x-kubernetes-validations: - - message: clusterCIDR is immutable - rule: self == oldSelf - serviceCIDR: - type: string - x-kubernetes-validations: - - message: serviceCIDR is immutable - rule: self == oldSelf - clusterDNS: - type: string - x-kubernetes-validations: - - message: clusterDNS is immutable - rule: self == oldSelf - serverArgs: - type: array - items: - type: string - agentArgs: - type: array - items: - type: string - tlsSANs: - type: array - items: - type: string - persistence: - type: object - properties: - type: - type: string - default: "ephermal" - storageClassName: - type: string - storageRequestSize: - type: string - addons: - type: array - items: - type: object - properties: - secretNamespace: - type: string - secretRef: - type: string - expose: - type: object - properties: - ingress: - type: object - properties: - enabled: - type: boolean - ingressClassName: - type: string - loadbalancer: - type: object - properties: - enabled: - type: boolean - nodePort: - type: object - properties: - enabled: - type: boolean - status: - type: object - properties: - overrideClusterCIDR: - type: boolean - clusterCIDR: - type: string - overrideServiceCIDR: - type: boolean - serviceCIDR: - type: string - clusterDNS: - type: string - tlsSANs: - type: array - items: - type: string - persistence: - type: object - properties: - type: - type: string - default: "ephermal" - storageClassName: - type: string - storageRequestSize: - type: string - scope: Cluster - names: - plural: clusters - singular: cluster - kind: Cluster \ No newline at end of file diff --git a/charts/k3k/crds/k3k.io_clusters.yaml b/charts/k3k/crds/k3k.io_clusters.yaml new file mode 100644 index 00000000..0071d179 --- /dev/null +++ b/charts/k3k/crds/k3k.io_clusters.yaml @@ -0,0 +1,206 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: clusters.k3k.io +spec: + group: k3k.io + names: + kind: Cluster + listKind: ClusterList + plural: clusters + singular: cluster + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + addons: + description: Addons is a list of secrets containing raw YAML which + will be deployed in the virtual K3k cluster on startup. + items: + properties: + secretNamespace: + type: string + secretRef: + type: string + type: object + type: array + agentArgs: + description: AgentArgs are the ordered key value pairs (e.x. "testArg", + "testValue") for the K3s pods running in agent mode. + items: + type: string + type: array + agents: + description: Agents is the number of K3s pods to run in agent (worker) + mode. + format: int32 + type: integer + x-kubernetes-validations: + - message: invalid value for agents + rule: self >= 0 + clusterCIDR: + description: ClusterCIDR is the CIDR range for the pods of the cluster. + Defaults to 10.42.0.0/16. + type: string + x-kubernetes-validations: + - message: clusterCIDR is immutable + rule: self == oldSelf + clusterDNS: + description: |- + ClusterDNS is the IP address for the coredns service. Needs to be in the range provided by ServiceCIDR or CoreDNS may not deploy. + Defaults to 10.43.0.10. + type: string + x-kubernetes-validations: + - message: clusterDNS is immutable + rule: self == oldSelf + expose: + description: |- + Expose contains options for exposing the apiserver inside/outside of the cluster. By default, this is only exposed as a + clusterIP which is relatively secure, but difficult to access outside of the cluster. + properties: + ingress: + properties: + enabled: + type: boolean + ingressClassName: + type: string + required: + - enabled + - ingressClassName + type: object + loadbalancer: + properties: + enabled: + type: boolean + required: + - enabled + type: object + nodePort: + properties: + enabled: + type: boolean + required: + - enabled + type: object + required: + - ingress + - loadbalancer + - nodePort + type: object + persistence: + description: |- + Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data + persistence is guaranteed, so restart of a virtual cluster pod may result in data loss without this field. + properties: + storageClassName: + type: string + storageRequestSize: + type: string + type: + default: ephemeral + description: Type can be ephermal, static, dynamic + type: string + required: + - type + type: object + serverArgs: + description: ServerArgs are the ordered key value pairs (e.x. "testArg", + "testValue") for the K3s pods running in server mode. + items: + type: string + type: array + servers: + description: Servers is the number of K3s pods to run in server (controlplane) + mode. + format: int32 + type: integer + x-kubernetes-validations: + - message: cluster must have at least one server + rule: self >= 1 + serviceCIDR: + description: ServiceCIDR is the CIDR range for the services in the + cluster. Defaults to 10.43.0.0/16. + type: string + x-kubernetes-validations: + - message: serviceCIDR is immutable + rule: self == oldSelf + tlsSANs: + description: TLSSANs are the subjectAlternativeNames for the certificate + the K3s server will use. + items: + type: string + type: array + token: + description: Token is the token used to join the worker nodes to the + cluster. + type: string + x-kubernetes-validations: + - message: token is immutable + rule: self == oldSelf + version: + description: Version is a string representing the Kubernetes version + to be used by the virtual nodes. + type: string + required: + - agents + - servers + - token + - version + type: object + status: + properties: + clusterCIDR: + type: string + clusterDNS: + type: string + persistence: + properties: + storageClassName: + type: string + storageRequestSize: + type: string + type: + default: ephemeral + description: Type can be ephermal, static, dynamic + type: string + required: + - type + type: object + serviceCIDR: + type: string + tlsSANs: + items: + type: string + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/cli/cmds/cluster/create.go b/cli/cmds/cluster/create.go index 508c0ea5..bd2b9b87 100644 --- a/cli/cmds/cluster/create.go +++ b/cli/cmds/cluster/create.go @@ -139,6 +139,7 @@ func create(clx *cli.Context) error { logrus.Infof("Creating a new cluster [%s]", name) cluster := newCluster( name, + cmds.Namespace(), token, int32(servers), int32(agents), @@ -226,10 +227,11 @@ func validateCreateFlags(clx *cli.Context) error { return nil } -func newCluster(name, token string, servers, agents int32, clusterCIDR, serviceCIDR string, serverArgs, agentArgs []string) *v1alpha1.Cluster { +func newCluster(name, namespace, token string, servers, agents int32, clusterCIDR, serviceCIDR string, serverArgs, agentArgs []string) *v1alpha1.Cluster { return &v1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: name, + Namespace: namespace, }, TypeMeta: metav1.TypeMeta{ Kind: "Cluster", diff --git a/cli/cmds/cluster/delete.go b/cli/cmds/cluster/delete.go index 5e0b8653..8876b939 100644 --- a/cli/cmds/cluster/delete.go +++ b/cli/cmds/cluster/delete.go @@ -40,7 +40,8 @@ func delete(clx *cli.Context) error { logrus.Infof("deleting [%s] cluster", name) cluster := v1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: name, + Namespace: cmds.Namespace(), }, } return ctrlClient.Delete(ctx, &cluster) diff --git a/cli/cmds/kubeconfig/kubeconfig.go b/cli/cmds/kubeconfig/kubeconfig.go index 6c6e1228..a78e15dc 100644 --- a/cli/cmds/kubeconfig/kubeconfig.go +++ b/cli/cmds/kubeconfig/kubeconfig.go @@ -114,9 +114,9 @@ func generate(clx *cli.Context) error { if err != nil { return err } - clusterKey := types.NamespacedName{ - Name: name, + Name: name, + Namespace: cmds.Namespace(), } if err := ctrlClient.Get(ctx, clusterKey, &cluster); err != nil { diff --git a/cli/cmds/root.go b/cli/cmds/root.go index b57a193c..36253542 100644 --- a/cli/cmds/root.go +++ b/cli/cmds/root.go @@ -5,9 +5,14 @@ import ( "github.com/urfave/cli" ) +const ( + defaultNamespace = "default" +) + var ( debug bool Kubeconfig string + namespace string CommonFlags = []cli.Flag{ cli.StringFlag{ Name: "kubeconfig", @@ -15,6 +20,11 @@ var ( Usage: "Kubeconfig path", Destination: &Kubeconfig, }, + cli.StringFlag{ + Name: "namespace", + Usage: "Namespace to create the k3k cluster in", + Destination: &namespace, + }, } ) @@ -40,3 +50,10 @@ func NewApp() *cli.App { return app } + +func Namespace() string { + if namespace == "" { + return defaultNamespace + } + return namespace +} diff --git a/main.go b/main.go index bf2548bd..ba7eb0e6 100644 --- a/main.go +++ b/main.go @@ -46,6 +46,10 @@ func main() { klog.Fatalf("Failed to add the new controller: %v", err) } + if err := cluster.AddPodController(ctx, mgr); err != nil { + klog.Fatalf("Failed to add the new cluster controller: %v", err) + } + if err := mgr.Start(ctx); err != nil { klog.Fatalf("Failed to start the manager: %v", err) } diff --git a/ops/build-crds b/ops/build-crds new file mode 100755 index 00000000..295bc4c8 --- /dev/null +++ b/ops/build-crds @@ -0,0 +1,8 @@ +#! /bin/sh + +cd $(dirname $0)/../ + +# This will return non-zero until all of our objects in ./pkg/apis can generate valid crds. +# allowDangerousTypes is needed for struct that use floats +controller-gen crd:generateEmbeddedObjectMeta=true,allowDangerousTypes=false paths=./pkg/apis/... output:crd:dir=./charts/k3k/crds + diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go index 73394256..58c8db2f 100644 --- a/pkg/apis/k3k.io/v1alpha1/types.go +++ b/pkg/apis/k3k.io/v1alpha1/types.go @@ -1,35 +1,67 @@ package v1alpha1 import ( + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:storageversion +// +kubebuilder:subresource:status type Cluster struct { metav1.ObjectMeta `json:"metadata,omitempty"` metav1.TypeMeta `json:",inline"` Spec ClusterSpec `json:"spec"` - Status ClusterStatus `json:"status"` + Status ClusterStatus `json:"status,omitempty"` } type ClusterSpec struct { - Version string `json:"version"` - Servers *int32 `json:"servers"` - Agents *int32 `json:"agents"` - Token string `json:"token"` - ClusterCIDR string `json:"clusterCIDR,omitempty"` - ServiceCIDR string `json:"serviceCIDR,omitempty"` - ClusterDNS string `json:"clusterDNS,omitempty"` - ServerArgs []string `json:"serverArgs,omitempty"` - AgentArgs []string `json:"agentArgs,omitempty"` - TLSSANs []string `json:"tlsSANs,omitempty"` - Addons []Addon `json:"addons,omitempty"` + // Version is a string representing the Kubernetes version to be used by the virtual nodes. + Version string `json:"version"` + // +kubebuilder:validation:XValidation:message="cluster must have at least one server",rule="self >= 1" + // Servers is the number of K3s pods to run in server (controlplane) mode. + Servers *int32 `json:"servers"` + // +kubebuilder:validation:XValidation:message="invalid value for agents",rule="self >= 0" + // Agents is the number of K3s pods to run in agent (worker) mode. + Agents *int32 `json:"agents"` + // +kubebuilder:validation:XValidation:message="token is immutable",rule="self == oldSelf" + // Token is the token used to join the worker nodes to the cluster. + Token string `json:"token"` + // +kubebuilder:validation:XValidation:message="clusterCIDR is immutable",rule="self == oldSelf" + // ClusterCIDR is the CIDR range for the pods of the cluster. Defaults to 10.42.0.0/16. + ClusterCIDR string `json:"clusterCIDR,omitempty"` + // +kubebuilder:validation:XValidation:message="serviceCIDR is immutable",rule="self == oldSelf" + // ServiceCIDR is the CIDR range for the services in the cluster. Defaults to 10.43.0.0/16. + ServiceCIDR string `json:"serviceCIDR,omitempty"` + // +kubebuilder:validation:XValidation:message="clusterDNS is immutable",rule="self == oldSelf" + // ClusterDNS is the IP address for the coredns service. Needs to be in the range provided by ServiceCIDR or CoreDNS may not deploy. + // Defaults to 10.43.0.10. + ClusterDNS string `json:"clusterDNS,omitempty"` + // ServerArgs are the ordered key value pairs (e.x. "testArg", "testValue") for the K3s pods running in server mode. + ServerArgs []string `json:"serverArgs,omitempty"` + // AgentArgs are the ordered key value pairs (e.x. "testArg", "testValue") for the K3s pods running in agent mode. + AgentArgs []string `json:"agentArgs,omitempty"` + // TLSSANs are the subjectAlternativeNames for the certificate the K3s server will use. + TLSSANs []string `json:"tlsSANs,omitempty"` + // Addons is a list of secrets containing raw YAML which will be deployed in the virtual K3k cluster on startup. + Addons []Addon `json:"addons,omitempty"` + // Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data + // persistence is guaranteed, so restart of a virtual cluster pod may result in data loss without this field. Persistence *PersistenceConfig `json:"persistence,omitempty"` - Expose *ExposeConfig `json:"expose,omitempty"` + // Expose contains options for exposing the apiserver inside/outside of the cluster. By default, this is only exposed as a + // clusterIP which is relatively secure, but difficult to access outside of the cluster. + Expose *ExposeConfig `json:"expose,omitempty"` +} + +type ClusterLimit struct { + // ServerLimit is the limits (cpu/mem) that apply to the server nodes + ServerLimit v1.ResourceList `json:"serverLimit,omitempty"` + // WorkerLimit is the limits (cpu/mem) that apply to the agent nodes + WorkerLimit v1.ResourceList `json:"workerLimit,omitempty"` } type Addon struct { @@ -48,6 +80,7 @@ type ClusterList struct { type PersistenceConfig struct { // Type can be ephermal, static, dynamic + // +kubebuilder:default="ephemeral" Type string `json:"type"` StorageClassName string `json:"storageClassName,omitempty"` StorageRequestSize string `json:"storageRequestSize,omitempty"` diff --git a/pkg/controller/cluster/agent/agent.go b/pkg/controller/cluster/agent/agent.go index 15cae241..1867f281 100644 --- a/pkg/controller/cluster/agent/agent.go +++ b/pkg/controller/cluster/agent/agent.go @@ -26,7 +26,12 @@ func (a *Agent) Deploy() *apps.Deployment { image := util.K3SImage(a.cluster) const name = "k3k-agent" - + selector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + "cluster": a.cluster.Name, + "type": "agent", + }, + } return &apps.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", @@ -35,23 +40,16 @@ func (a *Agent) Deploy() *apps.Deployment { ObjectMeta: metav1.ObjectMeta{ Name: a.cluster.Name + "-" + name, Namespace: util.ClusterNamespace(a.cluster), + Labels: selector.MatchLabels, }, Spec: apps.DeploymentSpec{ Replicas: a.cluster.Spec.Agents, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "cluster": a.cluster.Name, - "type": "agent", - }, - }, + Selector: &selector, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "cluster": a.cluster.Name, - "type": "agent", - }, + Labels: selector.MatchLabels, }, - Spec: a.podSpec(image, name, a.cluster.Spec.AgentArgs, false), + Spec: a.podSpec(image, name, a.cluster.Spec.AgentArgs, false, &selector), }, }, } @@ -60,6 +58,12 @@ func (a *Agent) Deploy() *apps.Deployment { func (a *Agent) StatefulAgent(cluster *v1alpha1.Cluster) *apps.StatefulSet { image := util.K3SImage(cluster) + selector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + "cluster": cluster.Name, + "type": "agent", + }, + } return &apps.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "Statefulset", @@ -68,16 +72,12 @@ func (a *Agent) StatefulAgent(cluster *v1alpha1.Cluster) *apps.StatefulSet { ObjectMeta: metav1.ObjectMeta{ Name: cluster.Name + "-" + agentName, Namespace: util.ClusterNamespace(cluster), + Labels: selector.MatchLabels, }, Spec: apps.StatefulSetSpec{ ServiceName: cluster.Name + "-" + agentName + "-headless", Replicas: cluster.Spec.Agents, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "cluster": cluster.Name, - "type": "agent", - }, - }, + Selector: &selector, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ { TypeMeta: metav1.TypeMeta{ @@ -120,20 +120,28 @@ func (a *Agent) StatefulAgent(cluster *v1alpha1.Cluster) *apps.StatefulSet { }, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "cluster": cluster.Name, - "type": "agent", - }, + Labels: selector.MatchLabels, }, - Spec: a.podSpec(image, agentName, cluster.Spec.AgentArgs, true), + Spec: a.podSpec(image, agentName, cluster.Spec.AgentArgs, true, &selector), }, }, } } -func (a *Agent) podSpec(image, name string, args []string, statefulSet bool) v1.PodSpec { +func (a *Agent) podSpec(image, name string, args []string, statefulSet bool, affinitySelector *metav1.LabelSelector) v1.PodSpec { + var limit v1.ResourceList args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...) podSpec := v1.PodSpec{ + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: affinitySelector, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, Volumes: []v1.Volume{ { Name: "config", @@ -185,6 +193,9 @@ func (a *Agent) podSpec(image, name string, args []string, statefulSet bool) v1. "/bin/k3s", }, Args: args, + Resources: v1.ResourceRequirements{ + Limits: limit, + }, VolumeMounts: []v1.VolumeMount{ { Name: "config", diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index fdf3014a..11441285 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -2,33 +2,22 @@ package cluster import ( "context" - "crypto/tls" - "crypto/x509" "errors" "fmt" - "net/url" "reflect" - "strings" "time" - certutil "github.com/rancher/dynamiclistener/cert" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/rancher/k3k/pkg/controller/cluster/agent" "github.com/rancher/k3k/pkg/controller/cluster/config" "github.com/rancher/k3k/pkg/controller/cluster/server" "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" - "github.com/rancher/k3k/pkg/controller/kubeconfig" "github.com/rancher/k3k/pkg/controller/util" - "github.com/sirupsen/logrus" - "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" - clientv3 "go.etcd.io/etcd/client/v3" - apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -76,49 +65,15 @@ func Add(ctx context.Context, mgr manager.Manager) error { return err } - if err := controller.Watch(&source.Kind{Type: &v1alpha1.Cluster{}}, &handler.EnqueueRequestForObject{}); err != nil { - return err - } - return controller.Watch(&source.Kind{Type: &v1.Pod{}}, - &handler.EnqueueRequestForOwner{IsController: true, OwnerType: &apps.StatefulSet{}}) + return controller.Watch(&source.Kind{Type: &v1alpha1.Cluster{}}, &handler.EnqueueRequestForObject{}) } func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { var ( - cluster v1alpha1.Cluster - podList v1.PodList - clusterName string + cluster v1alpha1.Cluster + podList v1.PodList ) - if req.Namespace != "" { - s := strings.Split(req.Namespace, "-") - if len(s) <= 1 { - return reconcile.Result{}, util.LogAndReturnErr("failed to get cluster namespace", nil) - } - - clusterName = s[1] - var cluster v1alpha1.Cluster - if err := c.Client.Get(ctx, types.NamespacedName{Name: clusterName}, &cluster); err != nil { - if !apierrors.IsNotFound(err) { - return reconcile.Result{}, err - } - } - matchingLabels := ctrlruntimeclient.MatchingLabels(map[string]string{"role": "server"}) - listOpts := &ctrlruntimeclient.ListOptions{Namespace: req.Namespace} - matchingLabels.ApplyToList(listOpts) - - if err := c.Client.List(ctx, &podList, listOpts); err != nil { - return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err) - } - for _, pod := range podList.Items { - klog.Infof("Handle etcd server pod [%s/%s]", pod.Namespace, pod.Name) - if err := c.handleServerPod(ctx, cluster, &pod); err != nil { - return reconcile.Result{}, util.LogAndReturnErr("failed to handle etcd pod", err) - } - } - - return reconcile.Result{}, nil - } if err := c.Client.Get(ctx, req.NamespacedName, &cluster); err != nil { return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err) @@ -132,17 +87,6 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request } } - // we create a namespace for each new cluster - var ns v1.Namespace - objKey := ctrlruntimeclient.ObjectKey{ - Name: util.ClusterNamespace(&cluster), - } - if err := c.Client.Get(ctx, objKey, &ns); err != nil { - if !apierrors.IsNotFound(err) { - return reconcile.Result{}, util.LogAndReturnErr("failed to get cluster namespace "+util.ClusterNamespace(&cluster), err) - } - } - klog.Infof("enqueue cluster [%s]", cluster.Name) if err := c.createCluster(ctx, &cluster); err != nil { return reconcile.Result{}, util.LogAndReturnErr("failed to create cluster", err) @@ -196,10 +140,6 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1 if err := c.Client.Update(ctx, cluster); err != nil { return util.LogAndReturnErr("failed to update cluster with persistence type", err) } - // create a new namespace for the cluster - if err := c.createNamespace(ctx, cluster); err != nil { - return util.LogAndReturnErr("failed to create ns", err) - } cluster.Status.ClusterCIDR = cluster.Spec.ClusterCIDR if cluster.Status.ClusterCIDR == "" { @@ -259,26 +199,6 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1 return c.Client.Update(ctx, cluster) } -func (c *ClusterReconciler) createNamespace(ctx context.Context, cluster *v1alpha1.Cluster) error { - // create a new namespace for the cluster - namespace := v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: util.ClusterNamespace(cluster), - }, - } - if err := controllerutil.SetControllerReference(cluster, &namespace, c.Scheme); err != nil { - return err - } - - if err := c.Client.Create(ctx, &namespace); err != nil { - if !apierrors.IsAlreadyExists(err) { - return util.LogAndReturnErr("failed to create ns", err) - } - } - - return nil -} - func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP string) error { // create init node config initServerConfig, err := config.Server(cluster, true, serviceIP) @@ -341,7 +261,7 @@ func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v objKey := ctrlruntimeclient.ObjectKey{ Namespace: util.ClusterNamespace(cluster), - Name: "k3k-server-service", + Name: util.ServerSvcName(cluster), } if err := c.Client.Get(ctx, objKey, &service); err != nil { return "", err @@ -400,7 +320,7 @@ func agentConfig(cluster *v1alpha1.Cluster, serviceIP string) v1.Secret { APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "k3k-agent-config", + Name: util.AgentConfigName(cluster), Namespace: util.ClusterNamespace(cluster), }, Data: map[string][]byte{ @@ -414,129 +334,6 @@ func agentData(serviceIP, token string) string { token: %s`, serviceIP, token) } -func (c *ClusterReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cluster, pod *v1.Pod) error { - if _, ok := pod.Labels["role"]; ok { - if pod.Labels["role"] != "server" { - return nil - } - } else { - return errors.New("server pod has no role label") - } - // if etcd pod is marked for deletion then we need to remove it from the etcd member list before deletion - if !pod.DeletionTimestamp.IsZero() { - // check if cluster is deleted then remove the finalizer from the pod - if cluster.Name == "" { - if controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) { - controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName) - if err := c.Client.Update(ctx, pod); err != nil { - return err - } - } - return nil - } - tlsConfig, err := c.getETCDTLS(&cluster) - if err != nil { - return err - } - // remove server from etcd - client, err := clientv3.New(clientv3.Config{ - Endpoints: []string{ - "https://k3k-server-service." + pod.Namespace + ":2379", - }, - TLS: tlsConfig, - }) - if err != nil { - return err - } - - if err := removePeer(ctx, client, pod.Name, pod.Status.PodIP); err != nil { - return err - } - // remove our finalizer from the list and update it. - if controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) { - controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName) - if err := c.Client.Update(ctx, pod); err != nil { - return err - } - } - } - if !controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) { - controllerutil.AddFinalizer(pod, etcdPodFinalizerName) - return c.Client.Update(ctx, pod) - } - - return nil -} - -// removePeer removes a peer from the cluster. The peer name and IP address must both match. -func removePeer(ctx context.Context, client *clientv3.Client, name, address string) error { - ctx, cancel := context.WithTimeout(ctx, memberRemovalTimeout) - defer cancel() - members, err := client.MemberList(ctx) - if err != nil { - return err - } - - for _, member := range members.Members { - if !strings.Contains(member.Name, name) { - continue - } - for _, peerURL := range member.PeerURLs { - u, err := url.Parse(peerURL) - if err != nil { - return err - } - if u.Hostname() == address { - logrus.Infof("Removing name=%s id=%d address=%s from etcd", member.Name, member.ID, address) - _, err := client.MemberRemove(ctx, member.ID) - if errors.Is(err, rpctypes.ErrGRPCMemberNotFound) { - return nil - } - return err - } - } - } - - return nil -} - -func (c *ClusterReconciler) getETCDTLS(cluster *v1alpha1.Cluster) (*tls.Config, error) { - klog.Infof("generating etcd TLS client certificate for cluster [%s]", cluster.Name) - token := cluster.Spec.Token - endpoint := "k3k-server-service." + util.ClusterNamespace(cluster) - var b *bootstrap.ControlRuntimeBootstrap - if err := retry.OnError(retry.DefaultBackoff, func(err error) bool { - return true - }, func() error { - var err error - b, err = bootstrap.DecodedBootstrap(token, endpoint) - return err - }); err != nil { - return nil, err - } - - etcdCert, etcdKey, err := kubeconfig.CreateClientCertKey("etcd-client", nil, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, 0, b.ETCDServerCA.Content, b.ETCDServerCAKey.Content) - if err != nil { - return nil, err - } - clientCert, err := tls.X509KeyPair(etcdCert, etcdKey) - if err != nil { - return nil, err - } - // create rootCA CertPool - cert, err := certutil.ParseCertsPEM([]byte(b.ETCDServerCA.Content)) - if err != nil { - return nil, err - } - pool := x509.NewCertPool() - pool.AddCert(cert[0]) - - return &tls.Config{ - RootCAs: pool, - Certificates: []tls.Certificate{clientCert}, - }, nil -} - func (c *ClusterReconciler) validate(cluster *v1alpha1.Cluster) error { if cluster.Name == ClusterInvalidName { return errors.New("invalid cluster name " + cluster.Name + " no action will be taken") diff --git a/pkg/controller/cluster/config/agent.go b/pkg/controller/cluster/config/agent.go index 2125ebcd..4f139de2 100644 --- a/pkg/controller/cluster/config/agent.go +++ b/pkg/controller/cluster/config/agent.go @@ -18,7 +18,7 @@ func Agent(cluster *v1alpha1.Cluster, serviceIP string) v1.Secret { APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "k3k-agent-config", + Name: util.AgentConfigName(cluster), Namespace: util.ClusterNamespace(cluster), }, Data: map[string][]byte{ diff --git a/pkg/controller/cluster/config/server.go b/pkg/controller/cluster/config/server.go index d1edc534..18993b80 100644 --- a/pkg/controller/cluster/config/server.go +++ b/pkg/controller/cluster/config/server.go @@ -1,22 +1,26 @@ package config import ( + "fmt" + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/rancher/k3k/pkg/controller/util" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// Server returns the secret for the server's config. Note that this doesn't set the ownerRef on the secret +// to tie it back to the cluster. func Server(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret, error) { - name := "k3k-server-config" + name := util.ServerConfigName(cluster) if init { - name = "k3k-init-server-config" + name = util.ServerInitConfigName(cluster) } cluster.Status.TLSSANs = append(cluster.Spec.TLSSANs, serviceIP, - "k3k-server-service", - "k3k-server-service."+util.ClusterNamespace(cluster), + util.ServerSvcName(cluster), + fmt.Sprintf("%s.%s", util.ServerSvcName(cluster), util.ClusterNamespace(cluster)), ) config := serverConfigData(serviceIP, cluster) diff --git a/pkg/controller/cluster/pod.go b/pkg/controller/cluster/pod.go new file mode 100644 index 00000000..0e57a223 --- /dev/null +++ b/pkg/controller/cluster/pod.go @@ -0,0 +1,214 @@ +package cluster + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/url" + "strings" + + certutil "github.com/rancher/dynamiclistener/cert" + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" + "github.com/rancher/k3k/pkg/controller/kubeconfig" + "github.com/rancher/k3k/pkg/controller/util" + "github.com/sirupsen/logrus" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + clientv3 "go.etcd.io/etcd/client/v3" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "k8s.io/klog" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +type PodReconciler struct { + Client ctrlruntimeclient.Client + Scheme *runtime.Scheme +} + +// Add adds a new controller to the manager +func AddPodController(ctx context.Context, mgr manager.Manager) error { + // initialize a new Reconciler + reconciler := PodReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + } + + // create a new controller and add it to the manager + //this can be replaced by the new builder functionality in controller-runtime + controller, err := controller.New(clusterController, mgr, controller.Options{ + Reconciler: &reconciler, + MaxConcurrentReconciles: maxConcurrentReconciles, + }) + if err != nil { + return err + } + + return controller.Watch(&source.Kind{Type: &v1.Pod{}}, + &handler.EnqueueRequestForOwner{IsController: true, OwnerType: &apps.StatefulSet{}}) +} + +func (p *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + s := strings.Split(req.Namespace, "-") + if len(s) <= 1 { + return reconcile.Result{}, util.LogAndReturnErr("failed to get cluster namespace", nil) + } + + clusterName := s[1] + var cluster v1alpha1.Cluster + if err := p.Client.Get(ctx, types.NamespacedName{Name: clusterName}, &cluster); err != nil { + if !apierrors.IsNotFound(err) { + return reconcile.Result{}, err + } + } + matchingLabels := ctrlruntimeclient.MatchingLabels(map[string]string{"role": "server"}) + listOpts := &ctrlruntimeclient.ListOptions{Namespace: req.Namespace} + matchingLabels.ApplyToList(listOpts) + + var podList v1.PodList + if err := p.Client.List(ctx, &podList, listOpts); err != nil { + return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err) + } + for _, pod := range podList.Items { + klog.Infof("Handle etcd server pod [%s/%s]", pod.Namespace, pod.Name) + if err := p.handleServerPod(ctx, cluster, &pod); err != nil { + return reconcile.Result{}, util.LogAndReturnErr("failed to handle etcd pod", err) + } + } + return reconcile.Result{}, nil +} + +func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cluster, pod *v1.Pod) error { + if _, ok := pod.Labels["role"]; ok { + if pod.Labels["role"] != "server" { + return nil + } + } else { + return fmt.Errorf("server pod has no role label") + } + // if etcd pod is marked for deletion then we need to remove it from the etcd member list before deletion + if !pod.DeletionTimestamp.IsZero() { + // check if cluster is deleted then remove the finalizer from the pod + if cluster.Name == "" { + if controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) { + controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName) + if err := p.Client.Update(ctx, pod); err != nil { + return err + } + } + return nil + } + tlsConfig, err := p.getETCDTLS(&cluster) + if err != nil { + return err + } + // remove server from etcd + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{ + fmt.Sprintf("https://%s.%s:2379", util.ServerSvcName(&cluster), pod.Namespace), + }, + TLS: tlsConfig, + }) + if err != nil { + return err + } + + if err := removePeer(ctx, client, pod.Name, pod.Status.PodIP); err != nil { + return err + } + // remove our finalizer from the list and update it. + if controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) { + controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName) + if err := p.Client.Update(ctx, pod); err != nil { + return err + } + } + } + if !controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) { + controllerutil.AddFinalizer(pod, etcdPodFinalizerName) + return p.Client.Update(ctx, pod) + } + + return nil +} + +func (p *PodReconciler) getETCDTLS(cluster *v1alpha1.Cluster) (*tls.Config, error) { + klog.Infof("generating etcd TLS client certificate for cluster [%s]", cluster.Name) + token := cluster.Spec.Token + endpoint := fmt.Sprintf("%s.%s", util.ServerSvcName(cluster), util.ClusterNamespace(cluster)) + var b *bootstrap.ControlRuntimeBootstrap + if err := retry.OnError(retry.DefaultBackoff, func(err error) bool { + return true + }, func() error { + var err error + b, err = bootstrap.DecodedBootstrap(token, endpoint) + return err + }); err != nil { + return nil, err + } + + etcdCert, etcdKey, err := kubeconfig.CreateClientCertKey("etcd-client", nil, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, 0, b.ETCDServerCA.Content, b.ETCDServerCAKey.Content) + if err != nil { + return nil, err + } + clientCert, err := tls.X509KeyPair(etcdCert, etcdKey) + if err != nil { + return nil, err + } + // create rootCA CertPool + cert, err := certutil.ParseCertsPEM([]byte(b.ETCDServerCA.Content)) + if err != nil { + return nil, err + } + pool := x509.NewCertPool() + pool.AddCert(cert[0]) + + return &tls.Config{ + RootCAs: pool, + Certificates: []tls.Certificate{clientCert}, + }, nil +} + +// removePeer removes a peer from the cluster. The peer name and IP address must both match. +func removePeer(ctx context.Context, client *clientv3.Client, name, address string) error { + ctx, cancel := context.WithTimeout(ctx, memberRemovalTimeout) + defer cancel() + members, err := client.MemberList(ctx) + if err != nil { + return err + } + + for _, member := range members.Members { + if !strings.Contains(member.Name, name) { + continue + } + for _, peerURL := range member.PeerURLs { + u, err := url.Parse(peerURL) + if err != nil { + return err + } + if u.Hostname() == address { + logrus.Infof("Removing name=%s id=%d address=%s from etcd", member.Name, member.ID, address) + _, err := client.MemberRemove(ctx, member.ID) + if errors.Is(err, rpctypes.ErrGRPCMemberNotFound) { + return nil + } + return err + } + } + } + + return nil +} diff --git a/pkg/controller/cluster/server/ingress.go b/pkg/controller/cluster/server/ingress.go index a42a1f47..154cb5f8 100644 --- a/pkg/controller/cluster/server/ingress.go +++ b/pkg/controller/cluster/server/ingress.go @@ -59,7 +59,7 @@ func (s *Server) ingressRules(addresses []string) []networkingv1.IngressRule { PathType: &pathTypePrefix, Backend: networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ - Name: "k3k-server-service", + Name: util.ServerSvcName(s.cluster), Port: networkingv1.ServiceBackendPort{ Number: serverPort, }, diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index f815ea1c..ffc1b34a 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -40,14 +40,24 @@ func New(cluster *v1alpha1.Cluster, client client.Client) *Server { } } -func (s *Server) podSpec(ctx context.Context, image, name string, persistent bool) v1.PodSpec { +func (s *Server) podSpec(ctx context.Context, image, name string, persistent bool, affinitySelector *metav1.LabelSelector) v1.PodSpec { podSpec := v1.PodSpec{ + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: affinitySelector, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, Volumes: []v1.Volume{ { Name: "initconfig", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: "k3k-init-server-config", + SecretName: util.ServerInitConfigName(s.cluster), Items: []v1.KeyToPath{ { Key: "config.yaml", @@ -61,7 +71,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo Name: "config", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: "k3k-server-config", + SecretName: util.ServerConfigName(s.cluster), Items: []v1.KeyToPath{ { Key: "config.yaml", @@ -313,7 +323,14 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster) volumeMounts = append(volumeMounts, volumeMount) } - podSpec := s.podSpec(ctx, image, name, persistent) + selector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + "cluster": cluster.Name, + "role": "server", + }, + } + + podSpec := s.podSpec(ctx, image, name, persistent, &selector) podSpec.Volumes = append(podSpec.Volumes, volumes...) podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...) @@ -325,23 +342,16 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster) ObjectMeta: metav1.ObjectMeta{ Name: cluster.Name + "-" + name, Namespace: util.ClusterNamespace(cluster), + Labels: selector.MatchLabels, }, Spec: apps.StatefulSetSpec{ - Replicas: &replicas, - ServiceName: cluster.Name + "-" + name + "-headless", - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "cluster": cluster.Name, - "role": "server", - }, - }, + Replicas: &replicas, + ServiceName: cluster.Name + "-" + name + "-headless", + Selector: &selector, VolumeClaimTemplates: pvClaims, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "cluster": cluster.Name, - "role": "server", - }, + Labels: selector.MatchLabels, }, Spec: podSpec, }, diff --git a/pkg/controller/cluster/server/service.go b/pkg/controller/cluster/server/service.go index fb99bcf3..8c003bcb 100644 --- a/pkg/controller/cluster/server/service.go +++ b/pkg/controller/cluster/server/service.go @@ -23,7 +23,7 @@ func (s *Server) Service(cluster *v1alpha1.Cluster) *v1.Service { APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "k3k-server-service", + Name: util.ServerSvcName(cluster), Namespace: util.ClusterNamespace(cluster), }, Spec: v1.ServiceSpec{ diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go index dbccf6e5..2b8de594 100644 --- a/pkg/controller/kubeconfig/kubeconfig.go +++ b/pkg/controller/kubeconfig/kubeconfig.go @@ -57,7 +57,7 @@ func (k *KubeConfig) Extract(ctx context.Context, client client.Client, cluster } // get the server service to extract the right IP nn = types.NamespacedName{ - Name: "k3k-server-service", + Name: util.ServerSvcName(cluster), Namespace: util.ClusterNamespace(cluster), } diff --git a/pkg/controller/util/util.go b/pkg/controller/util/util.go index 7b2f55b8..93523100 100644 --- a/pkg/controller/util/util.go +++ b/pkg/controller/util/util.go @@ -2,6 +2,7 @@ package util import ( "context" + "fmt" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" v1 "k8s.io/api/core/v1" @@ -21,7 +22,23 @@ const ( ) func ClusterNamespace(cluster *v1alpha1.Cluster) string { - return namespacePrefix + cluster.Name + return cluster.Namespace +} + +func ServerSvcName(cluster *v1alpha1.Cluster) string { + return fmt.Sprintf("k3k-%s-service", cluster.Name) +} + +func ServerConfigName(cluster *v1alpha1.Cluster) string { + return fmt.Sprintf("k3k-%s-server-config", cluster.Name) +} + +func ServerInitConfigName(cluster *v1alpha1.Cluster) string { + return fmt.Sprintf("k3k-init-%s-server-config", cluster.Name) +} + +func AgentConfigName(cluster *v1alpha1.Cluster) string { + return fmt.Sprintf("k3k-%s-agent-config", cluster.Name) } func K3SImage(cluster *v1alpha1.Cluster) string {