Assign pod's hostname if not assigned (#253)

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

.cr.yaml

@@ -0,0 +1 @@
+release-name-template: chart-{{ .Version }}

.github/workflows/chart.yml

@@ -1,31 +1,45 @@
+name: Chart
+
 on:
+  workflow_dispatch:
   push:
     tags:
     - "chart-*"
 
-env:
-    GH_TOKEN: ${{ github.token }}
-
-name: Chart
 permissions:
     contents: write
-    id-token: write
+
 jobs:
   chart-release:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
+      with:
+          fetch-depth: 0
 
-    - name: Package Chart
+    - name: Check tag
+      if: github.event_name == 'push'
       run: |
-        make package-chart;
+        pushed_tag=$(echo ${{ github.ref_name }} | sed "s/chart-//")
+        chart_tag=$(yq .version charts/k3k/Chart.yaml)
+        
+        echo pushed_tag=${pushed_tag} chart_tag=${chart_tag}
+        [ "${pushed_tag}" == "${chart_tag}" ]
 
-    - name: Release Chart
+    - name: Configure Git
       run: |
-        gh release upload ${{ github.ref_name }} deploy/*
-      
-    - name: Index Chart
-      run: |
-        make index-chart
-        
+        git config user.name "$GITHUB_ACTOR"
+        git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+    
+    - name: Install Helm
+      uses: azure/setup-helm@v4
+      env:
+        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+    - name: Run chart-releaser
+      uses: helm/chart-releaser-action@v1.6.0
+      with:
+        config: .cr.yaml
+      env:
+        CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/release.yml

@@ -63,7 +63,7 @@ jobs:
     - name: Check release tag
       id: release-tag
       run: |
-        CURRENT_TAG=$(git describe --tag --always)
+        CURRENT_TAG=$(git describe --tag --always --match="v[0-9]*")
 
         if git show-ref --tags ${CURRENT_TAG} --quiet; then
           echo "tag ${CURRENT_TAG} already exists";

.github/workflows/test.yaml

@@ -9,6 +9,23 @@ permissions:
     contents: read
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: golangci-lint
+      uses: golangci/golangci-lint-action@v6
+      with:
+        args: --timeout=5m
+        version: v1.60
+
   tests:
     runs-on: ubuntu-latest
 
@@ -23,19 +40,69 @@ jobs:
     - name: Check go modules
       run: |
         go mod tidy
+
+        git --no-pager diff go.mod go.sum
         test -z "$(git status --porcelain)"
 
     - name: Install tools
       run: |
         go install github.com/onsi/ginkgo/v2/ginkgo
-
-        # With Golang 1.22 we need to use the release-0.18 branch
-        go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.18
+        go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
 
         ENVTEST_BIN=$(setup-envtest use -p path)
         sudo mkdir -p /usr/local/kubebuilder/bin
         sudo cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin
 
     - name: Run tests
+      run: ginkgo -v -r --skip-file=tests
+  
+  tests-e2e:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+    
+    - name: Check go modules
       run: |
-        ginkgo run ./...
+        go mod tidy
+
+        git --no-pager diff go.mod go.sum
+        test -z "$(git status --porcelain)"
+
+    - name: Install Ginkgo
+      run: go install github.com/onsi/ginkgo/v2/ginkgo
+    
+    - name: Build
+      run: |
+        ./scripts/build
+
+        # add k3kcli to $PATH
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+    - name: Check k3kcli
+      run: k3kcli -v
+
+    - name: Run tests
+      run: ginkgo -v ./tests
+
+    - name: Archive k3s logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: k3s-logs
+        path: /tmp/k3s.log
+    
+    - name: Archive k3k logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: k3k-logs
+        path: /tmp/k3k.log

.golangci.yml

@@ -0,0 +1,12 @@
+linters:
+  enable:
+  # default linters
+  - errcheck
+  - gosimple
+  - govet
+  - ineffassign
+  - staticcheck
+  - unused
+
+  # extra
+  - misspell

.goreleaser.yaml

@@ -20,6 +20,9 @@ builds:
       - "amd64"
       - "arm64"
       - "s390x"
+    ldflags:
+      - -w -s # strip debug info and symbol table
+      - -X "github.com/rancher/k3k/pkg/buildinfo.Version={{ .Tag }}"
 
   - id: k3k-kubelet
     main: ./k3k-kubelet
@@ -32,7 +35,10 @@ builds:
       - "amd64"
       - "arm64"
       - "s390x"
-  
+    ldflags:
+      - -w -s # strip debug info and symbol table
+      - -X "github.com/rancher/k3k/pkg/buildinfo.Version={{ .Tag }}"
+
   - id: k3kcli
     main: ./cli
     binary: k3kcli
@@ -41,6 +47,9 @@ builds:
     goarch:
       - "amd64"
       - "arm64"
+    ldflags:
+      - -w -s # strip debug info and symbol table
+      - -X "github.com/rancher/k3k/pkg/buildinfo.Version={{ .Tag }}"
 
 archives:
   - format: binary

Dockerfile.dapper

@@ -1,4 +1,4 @@
-ARG GOLANG=rancher/hardened-build-base:v1.22.2b1
+ARG GOLANG=rancher/hardened-build-base:v1.23.4b1
 FROM ${GOLANG}
 
 ARG DAPPER_HOST_ARCH
@@ -17,15 +17,13 @@ ENV CONTROLLER_GEN_VERSION v0.14.0
 RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_GEN_VERSION}
 
 # Tool to setup the envtest framework to run the controllers integration tests
-# Note: With Golang 1.22 we need to use the release-0.18 branch
-ENV SETUP_ENVTEST_VERSION release-0.18
-RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@${SETUP_ENVTEST_VERSION} && \
+RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest && \
     ENVTEST_BIN=$(setup-envtest use -p path) && \
     mkdir -p /usr/local/kubebuilder/bin && \
     cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin
 
 ENV GO111MODULE on
-ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS GITHUB_TOKEN
+ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS GITHUB_TOKEN SKIP_TESTS GIT_TAG
 ENV DAPPER_SOURCE /go/src/github.com/rancher/k3k/
 ENV DAPPER_OUTPUT ./bin ./dist ./deploy ./charts
 ENV DAPPER_DOCKER_SOCKET true

README.md

@@ -1,143 +1,170 @@
-# K3K
+# K3k: Kubernetes in Kubernetes
 
 [![Experimental](https://img.shields.io/badge/status-experimental-orange.svg)](https://shields.io/)
+[![Go Report Card](https://goreportcard.com/badge/github.com/rancher/k3k)](https://goreportcard.com/report/github.com/rancher/k3k)
+![Tests](https://github.com/rancher/k3k/actions/workflows/test.yaml/badge.svg)
+![Build](https://github.com/rancher/k3k/actions/workflows/build.yml/badge.svg)
+
+
+K3k, Kubernetes in Kubernetes, is a tool that empowers you to create and manage isolated K3s clusters within your existing Kubernetes environment.  It enables efficient multi-tenancy, streamlined experimentation, and robust resource isolation, minimizing infrastructure costs by allowing you to run multiple lightweight Kubernetes clusters on the same physical host. K3k offers both "shared" mode, optimizing resource utilization, and "virtual" mode, providing complete isolation with dedicated K3s server pods. This allows you to access a full Kubernetes experience without the overhead of managing separate physical resources. 
+
+K3k integrates seamlessly with Rancher for simplified management of your embedded clusters.
 
-A Kubernetes in Kubernetes tool, k3k provides a way to run multiple embedded isolated k3s clusters on your kubernetes cluster.
 
 **Experimental Tool**
 
 This project is still under development and is considered experimental. It may have limitations, bugs, or changes. Please use with caution and report any issues you encounter. We appreciate your feedback as we continue to refine and improve this tool.
 
-## Example
 
-An example on creating a k3k cluster on an RKE2 host using k3kcli
+## Features and Benefits
 
-[![asciicast](https://asciinema.org/a/eYlc3dsL2pfP2B50i3Ea8MJJp.svg)](https://asciinema.org/a/eYlc3dsL2pfP2B50i3Ea8MJJp)
+- **Resource Isolation:** Ensure workload isolation and prevent resource contention between teams or applications. K3k allows you to define resource limits and quotas for each embedded cluster, guaranteeing that one team's workloads won't impact another's performance.
 
-## Architecture
+- **Simplified Multi-Tenancy:** Easily create dedicated Kubernetes environments for different users or projects, simplifying access control and management. Provide each team with their own isolated cluster, complete with its own namespaces, RBAC, and resource quotas, without the complexity of managing multiple physical clusters.
 
-K3K consists of a controller and a cli tool, the controller can be deployed via a helm chart and the cli can be downloaded from the releases page.
+- **Lightweight and Fast:** Leverage the lightweight nature of K3s to spin up and tear down clusters quickly, accelerating development and testing cycles. Spin up a new K3k cluster in seconds, test your application in a clean environment, and tear it down just as quickly, streamlining your CI/CD pipeline.
 
-### Controller
+- **Optimized Resource Utilization (Shared Mode):** Maximize your infrastructure investment by running multiple K3s clusters on the same physical host. K3k's shared mode allows you to efficiently share underlying resources, reducing overhead and minimizing costs.
 
-The K3K controller will watch a CRD called `clusters.k3k.io`. Once found, the controller will create a separate namespace and it will create a K3S cluster as specified in the spec of the object.
+- **Complete Isolation (Virtual Mode):** For enhanced security and isolation, K3k's virtual mode provides dedicated K3s server pods for each embedded cluster. This ensures complete separation of workloads and eliminates any potential resource contention or security risks.
 
-Each server and agent is created as a separate pod that runs in the new namespace.
-
-### CLI
-
-The CLI provides a quick and easy way to create K3K clusters using simple flags, and automatically exposes the K3K clusters so it's accessible via a kubeconfig.
-
-## Features
-
-### Isolation
-
-Each cluster runs in a sperate namespace that can be isolated via netowrk policies and RBAC rules, clusters also run in a sperate network namespace with flannel as the backend CNI. Finally, each cluster has a separate datastore which can be persisted.
-
-In addition, k3k offers a persistence feature that can help users to persist their datatstore, using dynamic storage class volumes.
-
-### Portability and Customization
-
-The "Cluster" object is considered the template of the cluster that you can re-use to spin up multiple clusters in a matter of seconds.
-
-K3K clusters use K3S internally and leverage all options that can be passed to K3S. Each cluster is exposed to the host cluster via NodePort, LoadBalancers, and Ingresses.
+- **Rancher Integration:** Simplify the management of your K3k clusters with Rancher. Leverage Rancher's intuitive UI and powerful features to monitor, manage, and scale your embedded clusters with ease.
 
 
-|                       | Separate Namespace  (for each tenant) | K3K                          | vcluster        | Separate Cluster (for each tenant) |
-|-----------------------|---------------------------------------|------------------------------|-----------------|------------------------------------|
-| Isolation             | Very weak                             | Very strong                  | strong          | Very strong                        |
-| Access for tenants    | Very restricted                       | Built-in k8s RBAC / Rancher  | Vclustser admin | Cluster admin                      |
-| Cost                  | Very cheap                            | Very cheap                   | cheap           | expensive                          |
-| Overhead              | Very low                              | Very low                     | Very low        | Very high                          |
-| Networking            | Shared                                | Separate                     | shared          | separate                           |
-| Cluster Configuration |                                       | Very easy                    | Very hard       |                                    |
+## Installation
+
+This section provides instructions on how to install K3k and the `k3kcli`.
+
+
+### Prerequisites
+
+* [Helm](https://helm.sh) must be installed to use the charts. Please refer to Helm's [documentation](https://helm.sh/docs) to get started.
+
+
+### Install the K3k controller
+
+1. Add the K3k Helm repository:
+
+   ```bash
+   helm repo add k3k https://rancher.github.io/k3k
+   helm repo update
+   ```
+
+2. Install the K3k controller:
+
+   ```bash
+   helm install --namespace k3k-system --create-namespace k3k k3k/k3k --devel
+   ```
+
+**NOTE:** K3k is currently under development, so the chart is marked as a development chart. This means you need to add the `--devel` flag to install it.  For production use, keep an eye on releases for stable versions. We recommend using the latest released version when possible.
+
+
+### Install the `k3kcli`
+
+The `k3kcli` provides a quick and easy way to create K3k clusters and automatically exposes them via a kubeconfig.
+
+To install it, simply download the latest available version for your architecture from the GitHub Releases page.
+
+For example, you can download the Linux amd64 version with:
+
+```
+wget -qO k3kcli https://github.com/rancher/k3k/releases/download/v0.2.2-rc4/k3kcli-linux-amd64 && \
+  chmod +x k3kcli && \
+  sudo mv k3kcli /usr/local/bin
+```
+
+You should now be able to run:
+```bash
+-> % k3kcli --version
+k3kcli Version: v0.2.2-rc4
+```
+
 
 ## Usage
 
-### Deploy K3K Controller
+This section provides examples of how to use the `k3kcli` to manage your K3k clusters.
 
-[Helm](https://helm.sh) must be installed to use the charts.  Please refer to
-Helm's [documentation](https://helm.sh/docs) to get started.
+**K3k operates within the context of your currently configured `kubectl` context.** This means that K3k respects the standard Kubernetes mechanisms for context configuration, including the `--kubeconfig` flag, the `$KUBECONFIG` environment variable, and the default `$HOME/.kube/config` file. Any K3k clusters you create will reside within the Kubernetes cluster that your `kubectl` is currently pointing to.
 
-Once Helm has been set up correctly, add the repo as follows:
 
-```sh
-  helm repo add k3k https://rancher.github.io/k3k
+### Creating a K3k Cluster
+
+To create a new K3k cluster, use the following command:
+
+```bash
+k3kcli cluster create mycluster
 ```
 
-If you had already added this repo earlier, run `helm repo update` to retrieve
-the latest versions of the packages.  You can then run `helm search repo
-k3k --devel` to see the charts.
+When the K3s server is ready, `k3kcli` will generate the necessary kubeconfig file and print instructions on how to use it.  
 
-To install the k3k chart:
+Here's an example of the output:
 
-```sh
-helm install my-k3k k3k/k3k --devel
+```bash
+INFO[0000] Creating a new cluster [mycluster]          
+INFO[0000] Extracting Kubeconfig for [mycluster] cluster 
+INFO[0000] waiting for cluster to be available..        
+INFO[0073] certificate CN=system:admin,O=system:masters signed by CN=k3s-client-ca@1738746570: notBefore=2025-02-05 09:09:30 +0000 UTC notAfter=2026-02-05 09:10:42 +0000 UTC 
+INFO[0073] You can start using the cluster with: 
+
+        export KUBECONFIG=/my/current/directory/mycluster-kubeconfig.yaml
+        kubectl cluster-info  
 ```
 
-To uninstall the chart:
+After exporting the generated kubeconfig, you should be able to reach your Kubernetes cluster:
 
-```sh
-helm delete my-k3k
+```bash
+export KUBECONFIG=/my/current/directory/mycluster-kubeconfig.yaml
+kubectl get nodes
+kubectl get pods -A
 ```
 
-**NOTE: Since k3k is still under development, the chart is marked as a development chart, this means that you need to add the `--devel` flag to install it.**
+You can also directly create a Cluster resource in some namespace, to create a K3k cluster:
 
-### Create a new cluster
-
-To create a new cluster you need to install and run the cli or create a cluster object, to install the cli:
-
-#### For linux and macOS
-
-1 - Donwload the binary, linux dowload url:
-```
-wget https://github.com/rancher/k3k/releases/download/v0.0.0-alpha2/k3kcli
-```
-macOS dowload url:
-```
-wget https://github.com/rancher/k3k/releases/download/v0.0.0-alpha2/k3kcli
-```
-Then copy to local bin
-```
-chmod +x k3kcli
-sudo cp k3kcli /usr/local/bin
+```bash
+kubectl apply -f - <<EOF
+apiVersion: k3k.io/v1alpha1
+kind: Cluster
+metadata:
+  name: mycluster
+  namespace: k3k-mycluster
+EOF
 ```
 
-#### For Windows 
+and use the `k3kcli` to retrieve the kubeconfig:
 
-1 - Download the Binary:
-Use PowerShell's Invoke-WebRequest cmdlet to download the binary:
-```powershel
-Invoke-WebRequest -Uri "https://github.com/rancher/k3k/releases/download/v0.0.0-alpha2/k3kcli-windows" -OutFile "k3kcli.exe"
-```
-2 - Copy the Binary to a Directory in PATH:
-To allow running the binary from any command prompt, you can copy it to a directory in your system's PATH. For example, copying it to C:\Users\<YourUsername>\bin (create this directory if it doesn't exist):
-```
-Copy-Item "k3kcli.exe" "C:\bin"
-```
-3 - Update Environment Variable (PATH):
-If you haven't already added `C:\bin` (or your chosen directory) to your PATH, you can do it through PowerShell:
-```
-setx PATH "C:\bin;%PATH%"
-```
-
-To create a new cluster you can use:
-
-```sh
-k3k cluster create --name example-cluster --token test
+```bash
+k3kcli kubeconfig generate --namespace k3k-mycluster --name mycluster 
 ```
 
 
-## Tests
+### Deleting a K3k Cluster
 
-To run the tests we use [Ginkgo](https://onsi.github.io/ginkgo/), and [`envtest`](https://book.kubebuilder.io/reference/envtest) for testing the controllers.
+To delete a K3k cluster, use the following command:
 
-Install the required binaries from `envtest` with [`setup-envtest`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/tools/setup-envtest), and then put them in the default path `/usr/local/kubebuilder/bin`:
-
-```
-ENVTEST_BIN=$(setup-envtest use -p path)
-sudo mkdir -p /usr/local/kubebuilder/bin
-sudo cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin
+```bash
+k3kcli cluster delete mycluster
 ```
 
-then run `ginkgo run ./...`.
+
+## Architecture
+
+For a detailed explanation of the K3k architecture, please refer to the [Architecture documentation](./docs/architecture.md).
+
+
+## Advanced Usage
+
+For more in-depth examples and information on advanced K3k usage, including details on shared vs. virtual modes, resource management, and other configuration options, please see the [Advanced Usage documentation](./docs/advanced-usage.md).
+
+
+## Development
+
+If you're interested in building K3k from source or contributing to the project, please refer to the [Development documentation](./docs/development.md).
+
+
+## License
+
+Copyright (c) 2014-2025 [SUSE](http://rancher.com/)
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0.
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

charts/k3k/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: k3k
 description: A Helm chart for K3K
 type: application
-version: 0.1.5-r5
-appVersion: 0.2.0
+version: 0.1.6-r1
+appVersion: v0.2.2-rc5

charts/k3k/crds/k3k.io_clusters.yaml

@@ -36,6 +36,7 @@ spec:
           metadata:
             type: object
           spec:
+            default: {}
             properties:
               addons:
                 description: Addons is a list of secrets containing raw YAML which
@@ -55,6 +56,7 @@ spec:
                   type: string
                 type: array
               agents:
+                default: 0
                 description: Agents is the number of K3s pods to run in agent (worker)
                   mode.
                 format: int32
@@ -109,8 +111,12 @@ spec:
                 properties:
                   ingress:
                     properties:
-                      enabled:
-                        type: boolean
+                      annotations:
+                        additionalProperties:
+                          type: string
+                        description: Annotations is a key value map that will enrich
+                          the Ingress annotations
+                        type: object
                       ingressClassName:
                         type: string
                     type: object
@@ -123,10 +129,24 @@ spec:
                     type: object
                   nodePort:
                     properties:
-                      enabled:
-                        type: boolean
-                    required:
-                    - enabled
+                      etcdPort:
+                        description: |-
+                          ETCDPort is the port on each node on which the ETCD service is exposed when type is NodePort.
+                          If not specified, a port will be allocated (default: 30000-32767)
+                        format: int32
+                        type: integer
+                      serverPort:
+                        description: |-
+                          ServerPort is the port on each node on which the K3s server service is exposed when type is NodePort.
+                          If not specified, a port will be allocated (default: 30000-32767)
+                        format: int32
+                        type: integer
+                      servicePort:
+                        description: |-
+                          ServicePort is the port on each node on which the K3s service is exposed when type is NodePort.
+                          If not specified, a port will be allocated (default: 30000-32767)
+                        format: int32
+                        type: integer
                     type: object
                 type: object
               mode:
@@ -152,6 +172,8 @@ spec:
                   In "shared" mode the node selector will be applied also to the workloads.
                 type: object
               persistence:
+                default:
+                  type: dynamic
                 description: |-
                   Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data
                   persistence is guaranteed, so restart of a virtual cluster pod may result in data loss without this field.
@@ -161,8 +183,8 @@ spec:
                   storageRequestSize:
                     type: string
                   type:
-                    default: ephemeral
-                    description: Type can be ephermal, static, dynamic
+                    default: dynamic
+                    description: PersistenceMode is the storage mode of a Cluster.
                     type: string
                 required:
                 - type
@@ -179,6 +201,7 @@ spec:
                   type: string
                 type: array
               servers:
+                default: 1
                 description: Servers is the number of K3s pods to run in server (controlplane)
                   mode.
                 format: int32
@@ -218,11 +241,6 @@ spec:
                 description: Version is a string representing the Kubernetes version
                   to be used by the virtual nodes.
                 type: string
-            required:
-            - agents
-            - mode
-            - servers
-            - version
             type: object
           status:
             properties:
@@ -230,6 +248,8 @@ spec:
                 type: string
               clusterDNS:
                 type: string
+              hostVersion:
+                type: string
               persistence:
                 properties:
                   storageClassName:
@@ -237,8 +257,8 @@ spec:
                   storageRequestSize:
                     type: string
                   type:
-                    default: ephemeral
-                    description: Type can be ephermal, static, dynamic
+                    default: dynamic
+                    description: PersistenceMode is the storage mode of a Cluster.
                     type: string
                 required:
                 - type
@@ -250,8 +270,6 @@ spec:
                   type: string
                 type: array
             type: object
-        required:
-        - spec
         type: object
     served: true
     storage: true

charts/k3k/templates/deployment.yaml

@@ -4,7 +4,7 @@ metadata:
   name: {{ include "k3k.fullname" . }}
   labels:
     {{- include "k3k.labels" . | nindent 4 }}
-  namespace: {{ .Values.namespace }}
+  namespace: {{ .Release.Namespace }}
 spec:
   replicas: {{ .Values.image.replicaCount }}
   selector:
@@ -16,14 +16,16 @@ spec:
         {{- include "k3k.selectorLabels" . | nindent 8 }}
     spec:
       containers:
-        - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+        - image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           name: {{ .Chart.Name }}
           env:
           - name: CLUSTER_CIDR
             value: {{ .Values.host.clusterCIDR }}
           - name: SHARED_AGENT_IMAGE
-            value: "{{ .Values.sharedAgent.image.repository }}:{{ .Values.sharedAgent.image.tag }}"
+            value: "{{ .Values.sharedAgent.image.repository }}:{{ default .Chart.AppVersion .Values.sharedAgent.image.tag }}"
+          - name: SHARED_AGENT_PULL_POLICY
+            value: {{ .Values.sharedAgent.image.pullPolicy }}
           ports:
           - containerPort: 8080
             name: https

charts/k3k/templates/namespace.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: {{ .Values.namespace }}

charts/k3k/templates/rbac.yaml

@@ -11,7 +11,7 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: {{ include "k3k.serviceAccountName" . }}
-    namespace: {{ .Values.namespace }}
+    namespace: {{ .Release.Namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

charts/k3k/templates/service.yaml

@@ -4,7 +4,7 @@ metadata:
   name: k3k-webhook
   labels:
     {{- include "k3k.labels" . | nindent 4 }}
-  namespace: {{ .Values.namespace }}
+  namespace: {{ .Release.Namespace }}
 spec:
   ports:
     - port: 443

charts/k3k/templates/serviceaccount.yaml

@@ -5,5 +5,5 @@ metadata:
   name: {{ include "k3k.serviceAccountName" . }}
   labels:
     {{- include "k3k.labels" . | nindent 4 }}
-  namespace: {{ .Values.namespace }}
-{{- end }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}

charts/k3k/values.yaml

@@ -1,19 +1,17 @@
 replicaCount: 1
-namespace: k3k-system
 
 image:
-  repository: rancher/k3k 
-  pullPolicy: Always
-  # Overrides the image tag whose default is the chart appVersion.
-  tag: "v0.2.1"
+  repository: rancher/k3k
+  tag: ""
+  pullPolicy: ""
 
 imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
 
 host:
-# clusterCIDR specifies the clusterCIDR that will be added to the default networkpolicy for clustersets, if not set
-# the controller will collect the PodCIDRs of all the nodes on the system.
+  # clusterCIDR specifies the clusterCIDR that will be added to the default networkpolicy for clustersets, if not set
+  # the controller will collect the PodCIDRs of all the nodes on the system.
   clusterCIDR: ""
 
 serviceAccount:
@@ -26,5 +24,6 @@ serviceAccount:
 # configuration related to the shared agent mode in k3k
 sharedAgent:
   image:
-    repository: "rancher/k3k"
-    tag: "k3k-kubelet-dev"
+    repository: "rancher/k3k-kubelet"
+    tag: ""
+    pullPolicy: ""

cli/cmds/cluster/cluster.go

@@ -1,33 +1,16 @@
 package cluster
 
 import (
-	"github.com/rancher/k3k/cli/cmds"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var subcommands = []cli.Command{
-	{
-		Name:            "create",
-		Usage:           "Create new cluster",
-		SkipFlagParsing: false,
-		SkipArgReorder:  true,
-		Action:          create,
-		Flags:           append(cmds.CommonFlags, clusterCreateFlags...),
-	},
-	{
-		Name:            "delete",
-		Usage:           "Delete an existing cluster",
-		SkipFlagParsing: false,
-		SkipArgReorder:  true,
-		Action:          delete,
-		Flags:           append(cmds.CommonFlags, clusterDeleteFlags...),
-	},
-}
-
-func NewCommand() cli.Command {
-	return cli.Command{
-		Name:        "cluster",
-		Usage:       "cluster command",
-		Subcommands: subcommands,
+func NewCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "cluster",
+		Usage: "cluster command",
+		Subcommands: []*cli.Command{
+			NewCreateCmd(),
+			NewDeleteCmd(),
+		},
 	}
 }

cli/cmds/cluster/create.go

@@ -11,21 +11,20 @@ import (
 
 	"github.com/rancher/k3k/cli/cmds"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	"github.com/rancher/k3k/pkg/controller"
 	k3kcluster "github.com/rancher/k3k/pkg/controller/cluster"
-	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/kubeconfig"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apiserver/pkg/authentication/user"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/util/retry"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -36,212 +35,137 @@ func init() {
 	_ = v1alpha1.AddToScheme(Scheme)
 }
 
-var (
-	name             string
-	token            string
-	clusterCIDR      string
-	serviceCIDR      string
-	servers          int64
-	agents           int64
-	serverArgs       cli.StringSlice
-	agentArgs        cli.StringSlice
-	persistenceType  string
-	storageClassName string
-	version          string
-	mode             string
-
-	clusterCreateFlags = []cli.Flag{
-		cli.StringFlag{
-			Name:        "name",
-			Usage:       "name of the cluster",
-			Destination: &name,
-		},
-		cli.Int64Flag{
-			Name:        "servers",
-			Usage:       "number of servers",
-			Destination: &servers,
-			Value:       1,
-		},
-		cli.Int64Flag{
-			Name:        "agents",
-			Usage:       "number of agents",
-			Destination: &agents,
-		},
-		cli.StringFlag{
-			Name:        "token",
-			Usage:       "token of the cluster",
-			Destination: &token,
-		},
-		cli.StringFlag{
-			Name:        "cluster-cidr",
-			Usage:       "cluster CIDR",
-			Destination: &clusterCIDR,
-		},
-		cli.StringFlag{
-			Name:        "service-cidr",
-			Usage:       "service CIDR",
-			Destination: &serviceCIDR,
-		},
-		cli.StringFlag{
-			Name:        "persistence-type",
-			Usage:       "Persistence mode for the nodes (ephermal, static, dynamic)",
-			Value:       server.EphermalNodesType,
-			Destination: &persistenceType,
-		},
-		cli.StringFlag{
-			Name:        "storage-class-name",
-			Usage:       "Storage class name for dynamic persistence type",
-			Destination: &storageClassName,
-		},
-		cli.StringSliceFlag{
-			Name:  "server-args",
-			Usage: "servers extra arguments",
-			Value: &serverArgs,
-		},
-		cli.StringSliceFlag{
-			Name:  "agent-args",
-			Usage: "agents extra arguments",
-			Value: &agentArgs,
-		},
-		cli.StringFlag{
-			Name:        "version",
-			Usage:       "k3s version",
-			Destination: &version,
-			Value:       "v1.26.1-k3s1",
-		},
-		cli.StringFlag{
-			Name:        "mode",
-			Usage:       "k3k mode type",
-			Destination: &mode,
-			Value:       "shared",
-		},
-	}
-)
-
-func create(clx *cli.Context) error {
-	ctx := context.Background()
-	if err := validateCreateFlags(); err != nil {
-		return err
-	}
-
-	restConfig, err := clientcmd.BuildConfigFromFlags("", cmds.Kubeconfig)
-	if err != nil {
-		return err
-	}
-
-	ctrlClient, err := client.New(restConfig, client.Options{
-		Scheme: Scheme,
-	})
-
-	if err != nil {
-		return err
-	}
-	if token != "" {
-		logrus.Infof("Creating cluster token secret")
-		obj := k3kcluster.TokenSecretObj(token, name, cmds.Namespace())
-		if err := ctrlClient.Create(ctx, &obj); err != nil {
-			return err
-		}
-	}
-	logrus.Infof("Creating a new cluster [%s]", name)
-	cluster := newCluster(
-		name,
-		cmds.Namespace(),
-		mode,
-		token,
-		int32(servers),
-		int32(agents),
-		clusterCIDR,
-		serviceCIDR,
-		serverArgs,
-		agentArgs,
-	)
-
-	cluster.Spec.Expose = &v1alpha1.ExposeConfig{
-		NodePort: &v1alpha1.NodePortConfig{
-			Enabled: true,
-		},
-	}
-
-	// add Host IP address as an extra TLS-SAN to expose the k3k cluster
-	url, err := url.Parse(restConfig.Host)
-	if err != nil {
-		return err
-	}
-	host := strings.Split(url.Host, ":")
-	cluster.Spec.TLSSANs = []string{host[0]}
-
-	if err := ctrlClient.Create(ctx, cluster); err != nil {
-		if apierrors.IsAlreadyExists(err) {
-			logrus.Infof("Cluster [%s] already exists", name)
-		} else {
-			return err
-		}
-	}
-
-	logrus.Infof("Extracting Kubeconfig for [%s] cluster", name)
-	cfg := &kubeconfig.KubeConfig{
-		CN:         controller.AdminCommonName,
-		ORG:        []string{user.SystemPrivilegedGroup},
-		ExpiryDate: 0,
-	}
-
-	logrus.Infof("waiting for cluster to be available..")
-
-	// retry every 5s for at most 2m, or 25 times
-	availableBackoff := wait.Backoff{
-		Duration: 5 * time.Second,
-		Cap:      2 * time.Minute,
-		Steps:    25,
-	}
-
-	var kubeconfig []byte
-	if err := retry.OnError(availableBackoff, apierrors.IsNotFound, func() error {
-		kubeconfig, err = cfg.Extract(ctx, ctrlClient, cluster, host[0])
-		return err
-	}); err != nil {
-		return err
-	}
-
-	pwd, err := os.Getwd()
-	if err != nil {
-		return err
-	}
-
-	logrus.Infof(`You can start using the cluster with: 
-
-	export KUBECONFIG=%s
-	kubectl cluster-info
-	`, filepath.Join(pwd, cluster.Name+"-kubeconfig.yaml"))
-
-	return os.WriteFile(cluster.Name+"-kubeconfig.yaml", kubeconfig, 0644)
+type CreateConfig struct {
+	token                string
+	clusterCIDR          string
+	serviceCIDR          string
+	servers              int
+	agents               int
+	serverArgs           cli.StringSlice
+	agentArgs            cli.StringSlice
+	persistenceType      string
+	storageClassName     string
+	version              string
+	mode                 string
+	kubeconfigServerHost string
 }
 
-func validateCreateFlags() error {
-	if persistenceType != server.EphermalNodesType &&
-		persistenceType != server.DynamicNodesType {
-		return errors.New("invalid persistence type")
-	}
-	if name == "" {
-		return errors.New("empty cluster name")
-	}
-	if name == k3kcluster.ClusterInvalidName {
-		return errors.New("invalid cluster name")
-	}
-	if servers <= 0 {
-		return errors.New("invalid number of servers")
-	}
-	if cmds.Kubeconfig == "" && os.Getenv("KUBECONFIG") == "" {
-		return errors.New("empty kubeconfig")
-	}
-	if mode != "shared" && mode != "virtual" {
-		return errors.New(`mode should be one of "shared" or "virtual"`)
-	}
+func NewCreateCmd() *cli.Command {
+	createConfig := &CreateConfig{}
+	createFlags := NewCreateFlags(createConfig)
 
-	return nil
+	return &cli.Command{
+		Name:            "create",
+		Usage:           "Create new cluster",
+		UsageText:       "k3kcli cluster create [command options] NAME",
+		Action:          createAction(createConfig),
+		Flags:           append(cmds.CommonFlags, createFlags...),
+		HideHelpCommand: true,
+	}
 }
 
-func newCluster(name, namespace, mode, token string, servers, agents int32, clusterCIDR, serviceCIDR string, serverArgs, agentArgs []string) *v1alpha1.Cluster {
+func createAction(config *CreateConfig) cli.ActionFunc {
+	return func(clx *cli.Context) error {
+		ctx := context.Background()
+
+		if clx.NArg() != 1 {
+			return cli.ShowSubcommandHelp(clx)
+		}
+
+		name := clx.Args().First()
+		if name == k3kcluster.ClusterInvalidName {
+			return errors.New("invalid cluster name")
+		}
+
+		restConfig, err := clientcmd.BuildConfigFromFlags("", cmds.Kubeconfig)
+		if err != nil {
+			return err
+		}
+
+		ctrlClient, err := client.New(restConfig, client.Options{
+			Scheme: Scheme,
+		})
+		if err != nil {
+			return err
+		}
+
+		if config.token != "" {
+			logrus.Infof("Creating cluster token secret")
+			obj := k3kcluster.TokenSecretObj(config.token, name, cmds.Namespace())
+			if err := ctrlClient.Create(ctx, &obj); err != nil {
+				return err
+			}
+		}
+
+		logrus.Infof("Creating a new cluster [%s]", name)
+
+		cluster := newCluster(name, cmds.Namespace(), config)
+
+		cluster.Spec.Expose = &v1alpha1.ExposeConfig{
+			NodePort: &v1alpha1.NodePortConfig{},
+		}
+
+		// add Host IP address as an extra TLS-SAN to expose the k3k cluster
+		url, err := url.Parse(restConfig.Host)
+		if err != nil {
+			return err
+		}
+		host := strings.Split(url.Host, ":")
+		if config.kubeconfigServerHost != "" {
+			host = []string{config.kubeconfigServerHost}
+		}
+		cluster.Spec.TLSSANs = []string{host[0]}
+
+		if err := ctrlClient.Create(ctx, cluster); err != nil {
+			if apierrors.IsAlreadyExists(err) {
+				logrus.Infof("Cluster [%s] already exists", name)
+			} else {
+				return err
+			}
+		}
+
+		logrus.Infof("Extracting Kubeconfig for [%s] cluster", name)
+
+		logrus.Infof("waiting for cluster to be available..")
+
+		// retry every 5s for at most 2m, or 25 times
+		availableBackoff := wait.Backoff{
+			Duration: 5 * time.Second,
+			Cap:      2 * time.Minute,
+			Steps:    25,
+		}
+
+		cfg := kubeconfig.New()
+
+		var kubeconfig *clientcmdapi.Config
+		if err := retry.OnError(availableBackoff, apierrors.IsNotFound, func() error {
+			kubeconfig, err = cfg.Extract(ctx, ctrlClient, cluster, host[0])
+			return err
+		}); err != nil {
+			return err
+		}
+
+		pwd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+
+		logrus.Infof(`You can start using the cluster with: 
+	
+		export KUBECONFIG=%s
+		kubectl cluster-info
+		`, filepath.Join(pwd, cluster.Name+"-kubeconfig.yaml"))
+
+		kubeconfigData, err := clientcmd.Write(*kubeconfig)
+		if err != nil {
+			return err
+		}
+
+		return os.WriteFile(cluster.Name+"-kubeconfig.yaml", kubeconfigData, 0644)
+	}
+}
+
+func newCluster(name, namespace string, config *CreateConfig) *v1alpha1.Cluster {
 	cluster := &v1alpha1.Cluster{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -252,25 +176,29 @@ func newCluster(name, namespace, mode, token string, servers, agents int32, clus
 			APIVersion: "k3k.io/v1alpha1",
 		},
 		Spec: v1alpha1.ClusterSpec{
-			Servers:     &servers,
-			Agents:      &agents,
-			ClusterCIDR: clusterCIDR,
-			ServiceCIDR: serviceCIDR,
-			ServerArgs:  serverArgs,
-			AgentArgs:   agentArgs,
-			Version:     version,
-			Mode:        v1alpha1.ClusterMode(mode),
-			Persistence: &v1alpha1.PersistenceConfig{
-				Type:             persistenceType,
-				StorageClassName: storageClassName,
+			Servers:     ptr.To(int32(config.servers)),
+			Agents:      ptr.To(int32(config.agents)),
+			ClusterCIDR: config.clusterCIDR,
+			ServiceCIDR: config.serviceCIDR,
+			ServerArgs:  config.serverArgs.Value(),
+			AgentArgs:   config.agentArgs.Value(),
+			Version:     config.version,
+			Mode:        v1alpha1.ClusterMode(config.mode),
+			Persistence: v1alpha1.PersistenceConfig{
+				Type:             v1alpha1.PersistenceMode(config.persistenceType),
+				StorageClassName: ptr.To(config.storageClassName),
 			},
 		},
 	}
-	if token != "" {
+	if config.storageClassName == "" {
+		cluster.Spec.Persistence.StorageClassName = nil
+	}
+	if config.token != "" {
 		cluster.Spec.TokenSecretRef = &v1.SecretReference{
 			Name:      k3kcluster.TokenSecretName(name),
 			Namespace: namespace,
 		}
 	}
+
 	return cluster
 }

cli/cmds/cluster/create_flags.go

@@ -0,0 +1,98 @@
+package cluster
+
+import (
+	"errors"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/urfave/cli/v2"
+)
+
+func NewCreateFlags(config *CreateConfig) []cli.Flag {
+	return []cli.Flag{
+		&cli.IntFlag{
+			Name:        "servers",
+			Usage:       "number of servers",
+			Destination: &config.servers,
+			Value:       1,
+			Action: func(ctx *cli.Context, value int) error {
+				if value <= 0 {
+					return errors.New("invalid number of servers")
+				}
+				return nil
+			},
+		},
+		&cli.IntFlag{
+			Name:        "agents",
+			Usage:       "number of agents",
+			Destination: &config.agents,
+		},
+		&cli.StringFlag{
+			Name:        "token",
+			Usage:       "token of the cluster",
+			Destination: &config.token,
+		},
+		&cli.StringFlag{
+			Name:        "cluster-cidr",
+			Usage:       "cluster CIDR",
+			Destination: &config.clusterCIDR,
+		},
+		&cli.StringFlag{
+			Name:        "service-cidr",
+			Usage:       "service CIDR",
+			Destination: &config.serviceCIDR,
+		},
+		&cli.StringFlag{
+			Name:        "persistence-type",
+			Usage:       "persistence mode for the nodes (ephemeral, static, dynamic)",
+			Value:       string(v1alpha1.DynamicNodesType),
+			Destination: &config.persistenceType,
+			Action: func(ctx *cli.Context, value string) error {
+				switch v1alpha1.PersistenceMode(value) {
+				case v1alpha1.EphemeralNodeType, v1alpha1.DynamicNodesType:
+					return nil
+				default:
+					return errors.New(`persistence-type should be one of "ephemeral", "static" or "dynamic"`)
+				}
+			},
+		},
+		&cli.StringFlag{
+			Name:        "storage-class-name",
+			Usage:       "storage class name for dynamic persistence type",
+			Destination: &config.storageClassName,
+		},
+		&cli.StringSliceFlag{
+			Name:  "server-args",
+			Usage: "servers extra arguments",
+			Value: &config.serverArgs,
+		},
+		&cli.StringSliceFlag{
+			Name:  "agent-args",
+			Usage: "agents extra arguments",
+			Value: &config.agentArgs,
+		},
+		&cli.StringFlag{
+			Name:        "version",
+			Usage:       "k3s version",
+			Destination: &config.version,
+		},
+		&cli.StringFlag{
+			Name:        "mode",
+			Usage:       "k3k mode type",
+			Destination: &config.mode,
+			Value:       "shared",
+			Action: func(ctx *cli.Context, value string) error {
+				switch value {
+				case string(v1alpha1.VirtualClusterMode), string(v1alpha1.SharedClusterMode):
+					return nil
+				default:
+					return errors.New(`mode should be one of "shared" or "virtual"`)
+				}
+			},
+		},
+		&cli.StringFlag{
+			Name:        "kubeconfig-server",
+			Usage:       "override the kubeconfig server host",
+			Destination: &config.kubeconfigServerHost,
+		},
+	}
+}

cli/cmds/cluster/delete.go

@@ -2,29 +2,41 @@ package cluster
 
 import (
 	"context"
+	"errors"
 
 	"github.com/rancher/k3k/cli/cmds"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	k3kcluster "github.com/rancher/k3k/pkg/controller/cluster"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-var (
-	clusterDeleteFlags = []cli.Flag{
-		cli.StringFlag{
-			Name:        "name",
-			Usage:       "name of the cluster",
-			Destination: &name,
-		},
+func NewDeleteCmd() *cli.Command {
+	return &cli.Command{
+		Name:            "delete",
+		Usage:           "Delete an existing cluster",
+		UsageText:       "k3kcli cluster delete [command options] NAME",
+		Action:          delete,
+		Flags:           cmds.CommonFlags,
+		HideHelpCommand: true,
 	}
-)
+}
 
 func delete(clx *cli.Context) error {
 	ctx := context.Background()
 
+	if clx.NArg() != 1 {
+		return cli.ShowSubcommandHelp(clx)
+	}
+
+	name := clx.Args().First()
+	if name == k3kcluster.ClusterInvalidName {
+		return errors.New("invalid cluster name")
+	}
+
 	restConfig, err := clientcmd.BuildConfigFromFlags("", cmds.Kubeconfig)
 	if err != nil {
 		return err
@@ -38,6 +50,7 @@ func delete(clx *cli.Context) error {
 	}
 
 	logrus.Infof("deleting [%s] cluster", name)
+
 	cluster := v1alpha1.Cluster{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,

cli/cmds/kubeconfig/kubeconfig.go

@@ -14,13 +14,14 @@ import (
 	"github.com/rancher/k3k/pkg/controller/certs"
 	"github.com/rancher/k3k/pkg/controller/kubeconfig"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apiserver/pkg/authentication/user"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -38,55 +39,61 @@ var (
 	altNames                cli.StringSlice
 	expirationDays          int64
 	configName              string
+	kubeconfigServerHost    string
 	generateKubeconfigFlags = []cli.Flag{
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "name",
 			Usage:       "cluster name",
 			Destination: &name,
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "config-name",
 			Usage:       "the name of the generated kubeconfig file",
 			Destination: &configName,
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "cn",
 			Usage:       "Common name (CN) of the generated certificates for the kubeconfig",
 			Destination: &cn,
 			Value:       controller.AdminCommonName,
 		},
-		cli.StringSliceFlag{
+		&cli.StringSliceFlag{
 			Name:  "org",
 			Usage: "Organization name (ORG) of the generated certificates for the kubeconfig",
 			Value: &org,
 		},
-		cli.StringSliceFlag{
+		&cli.StringSliceFlag{
 			Name:  "altNames",
 			Usage: "altNames of the generated certificates for the kubeconfig",
 			Value: &altNames,
 		},
-		cli.Int64Flag{
+		&cli.Int64Flag{
 			Name:        "expiration-days",
 			Usage:       "Expiration date of the certificates used for the kubeconfig",
 			Destination: &expirationDays,
 			Value:       356,
 		},
+		&cli.StringFlag{
+			Name:        "kubeconfig-server",
+			Usage:       "override the kubeconfig server host",
+			Destination: &kubeconfigServerHost,
+			Value:       "",
+		},
 	}
 )
 
-var subcommands = []cli.Command{
+var subcommands = []*cli.Command{
 	{
 		Name:            "generate",
 		Usage:           "Generate kubeconfig for clusters",
 		SkipFlagParsing: false,
-		SkipArgReorder:  true,
 		Action:          generate,
 		Flags:           append(cmds.CommonFlags, generateKubeconfigFlags...),
 	},
 }
 
-func NewCommand() cli.Command {
-	return cli.Command{
+func NewCommand() *cli.Command {
+	return &cli.Command{
 		Name:        "kubeconfig",
 		Usage:       "Manage kubeconfig for clusters",
 		Subcommands: subcommands,
@@ -122,28 +129,38 @@ func generate(clx *cli.Context) error {
 		return err
 	}
 	host := strings.Split(url.Host, ":")
-
-	certAltNames := certs.AddSANs(altNames)
-	if org == nil {
-		org = cli.StringSlice{user.SystemPrivilegedGroup}
-	}
-	cfg := kubeconfig.KubeConfig{
-		CN:         cn,
-		ORG:        org,
-		ExpiryDate: time.Hour * 24 * time.Duration(expirationDays),
-		AltNames:   certAltNames,
-	}
-	logrus.Infof("waiting for cluster to be available..")
-	var kubeconfig []byte
-	if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
-		kubeconfig, err = cfg.Extract(ctx, ctrlClient, &cluster, host[0])
+	if kubeconfigServerHost != "" {
+		host = []string{kubeconfigServerHost}
+		err := altNames.Set(kubeconfigServerHost)
 		if err != nil {
 			return err
 		}
-		return nil
+	}
+
+	certAltNames := certs.AddSANs(altNames.Value())
+
+	orgs := org.Value()
+	if orgs == nil {
+		orgs = []string{user.SystemPrivilegedGroup}
+	}
+
+	cfg := kubeconfig.KubeConfig{
+		CN:         cn,
+		ORG:        orgs,
+		ExpiryDate: time.Hour * 24 * time.Duration(expirationDays),
+		AltNames:   certAltNames,
+	}
+
+	logrus.Infof("waiting for cluster to be available..")
+
+	var kubeconfig *clientcmdapi.Config
+	if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
+		kubeconfig, err = cfg.Extract(ctx, ctrlClient, &cluster, host[0])
+		return err
 	}); err != nil {
 		return err
 	}
+
 	pwd, err := os.Getwd()
 	if err != nil {
 		return err
@@ -159,5 +176,10 @@ func generate(clx *cli.Context) error {
 	kubectl cluster-info
 	`, filepath.Join(pwd, configName))
 
-	return os.WriteFile(configName, kubeconfig, 0644)
+	kubeconfigData, err := clientcmd.Write(*kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(configName, kubeconfigData, 0644)
 }

cli/cmds/root.go

@@ -1,8 +1,10 @@
 package cmds
 
 import (
+	"os"
+
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -14,15 +16,16 @@ var (
 	Kubeconfig  string
 	namespace   string
 	CommonFlags = []cli.Flag{
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "kubeconfig",
-			EnvVar:      "KUBECONFIG",
-			Usage:       "Kubeconfig path",
+			EnvVars:     []string{"KUBECONFIG"},
+			Usage:       "kubeconfig path",
 			Destination: &Kubeconfig,
+			Value:       os.Getenv("HOME") + "/.kube/config",
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "namespace",
-			Usage:       "Namespace to create the k3k cluster in",
+			Usage:       "namespace to create the k3k cluster in",
 			Destination: &namespace,
 		},
 	}
@@ -33,11 +36,11 @@ func NewApp() *cli.App {
 	app.Name = "k3kcli"
 	app.Usage = "CLI for K3K"
 	app.Flags = []cli.Flag{
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "Turn on debug logs",
 			Destination: &debug,
-			EnvVar:      "K3K_DEBUG",
+			EnvVars:     []string{"K3K_DEBUG"},
 		},
 	}
 

cli/main.go

@@ -1,28 +1,28 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/rancher/k3k/cli/cmds"
 	"github.com/rancher/k3k/cli/cmds/cluster"
 	"github.com/rancher/k3k/cli/cmds/kubeconfig"
+	"github.com/rancher/k3k/pkg/buildinfo"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-const (
-	program   = "k3kcli"
-	version   = "dev"
-	gitCommit = "HEAD"
+	"github.com/urfave/cli/v2"
 )
 
 func main() {
 	app := cmds.NewApp()
-	app.Commands = []cli.Command{
+	app.Version = buildinfo.Version
+	cli.VersionPrinter = func(cCtx *cli.Context) {
+		fmt.Println("k3kcli Version: " + buildinfo.Version)
+	}
+
+	app.Commands = []*cli.Command{
 		cluster.NewCommand(),
 		kubeconfig.NewCommand(),
 	}
-	app.Version = version + " (" + gitCommit + ")"
 
 	if err := app.Run(os.Args); err != nil {
 		logrus.Fatal(err)

docs/advanced-usage.md

@@ -0,0 +1,114 @@
+# Advanced Usage
+
+This document provides advanced usage information for k3k, including detailed use cases and explanations of the `Cluster` resource fields for customization.
+
+## Customizing the Cluster Resource
+
+The `Cluster` resource provides a variety of fields for customizing the behavior of your virtual clusters. You can check the [CRD documentation](./crds/crd-docs.md) for the full specs.
+
+**Note:** Most of these customization options can also be configured using the `k3kcli` tool. Refer to the `k3kcli` documentation for more details.
+
+
+
+This example creates a "shared" mode K3k cluster with:
+
+- 3 servers
+- K3s version v1.31.3-k3s1
+- Custom network configuration 
+- Deployment on specific nodes with the `nodeSelector`
+- `kube-api` exposed using an ingress
+- Custom K3s `serverArgs`
+- ETCD data persisted using a `PVC`
+
+
+```yaml
+apiVersion: k3k.io/v1alpha1
+kind: Cluster
+metadata:
+  name: my-virtual-cluster
+  namespace: my-namespace
+spec:
+  mode: shared
+  version: v1.31.3-k3s1
+  servers: 3
+  tlsSANs:
+    - my-cluster.example.com
+  nodeSelector:
+    disktype: ssd
+  expose:
+    ingress:
+      ingressClassName: nginx
+      annotations:
+        nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+        nginx.ingress.kubernetes.io/backend-protocol: "true"
+        nginx.ingress.kubernetes.io/ssl-redirect: "HTTPS"
+  clusterCIDR: 10.42.0.0/16
+  serviceCIDR: 10.43.0.0/16
+  clusterDNS: 10.43.0.10
+  serverArgs:
+  - --tls-san=my-cluster.example.com
+  persistence:
+    type: dynamic
+    storageClassName: local-path
+```
+
+
+### `mode`
+
+The `mode` field specifies the cluster provisioning mode, which can be either `shared` or `virtual`. The default mode is `shared`.
+
+* **`shared` mode:** In this mode, the virtual cluster shares the host cluster's resources and networking. This mode is suitable for lightweight workloads and development environments where isolation is not a primary concern.
+* **`virtual` mode:** In this mode, the virtual cluster runs as a separate K3s cluster within the host cluster. This mode provides stronger isolation and is suitable for production workloads or when dedicated resources are required.
+
+
+### `version`
+
+The `version` field specifies the Kubernetes version to be used by the virtual nodes. If not specified, K3k will use the same K3s version as the host cluster. For example, if the host cluster is running Kubernetes v1.31.3, K3k will use the corresponding K3s version (e.g., `v1.31.3-k3s1`).
+
+
+### `servers`
+
+The `servers` field specifies the number of K3s server nodes to deploy for the virtual cluster. The default value is 1.
+
+
+### `agents`
+
+The `agents` field specifies the number of K3s agent nodes to deploy for the virtual cluster. The default value is 0.
+
+**Note:** In `shared` mode, this field is ignored, as the Virtual Kubelet acts as the agent, and there are no K3s worker nodes.
+
+
+### `nodeSelector`
+
+The `nodeSelector` field allows you to specify a node selector that will be applied to all server/agent pods. In `shared` mode, the node selector will also be applied to the workloads.
+
+
+### `expose`
+
+The `expose` field contains options for exposing the API server of the virtual cluster. By default, the API server is only exposed as a `ClusterIP`, which is relatively secure but difficult to access from outside the cluster.
+
+You can use the `expose` field to enable exposure via `NodePort`, `LoadBalancer`, or `Ingress`.
+
+In this example we are exposing the Cluster with a Nginx ingress-controller, that has to be configured with the `--enable-ssl-passthrough` flag.
+
+
+### `clusterCIDR`
+
+The `clusterCIDR` field specifies the CIDR range for the pods of the cluster. The default value is `10.42.0.0/16`.
+
+
+### `serviceCIDR`
+
+The `serviceCIDR` field specifies the CIDR range for the services in the cluster. The default value is `10.43.0.0/16`.
+
+**Note:** In `shared` mode, the `serviceCIDR` should match the host cluster's `serviceCIDR` to prevent conflicts.
+
+
+### `clusterDNS`
+
+The `clusterDNS` field specifies the IP address for the CoreDNS service. It needs to be in the range provided by `serviceCIDR`. The default value is `10.43.0.10`.
+
+
+### `serverArgs`
+
+The `serverArgs` field allows you to specify additional arguments to be passed to the K3s server pods.

docs/architecture.md

@@ -0,0 +1,121 @@
+# Architecture
+
+Virtual Clusters are isolated Kubernetes clusters provisioned on a physical cluster. K3k leverages [K3s](https://k3s.io/) as the control plane of the Kubernetes cluster because of its lightweight footprint.
+
+K3k provides two modes of deploying virtual clusters: the "shared" mode (default), and "virtual".
+
+
+## Shared Mode
+
+The default `shared` mode uses a K3s server as control plane with an [agentless servers configuration](https://docs.k3s.io/advanced#running-agentless-servers-experimental). With this option enabled, the servers do not run the kubelet, container runtime, or CNI. The server uses a [Virtual Kubelet](https://virtual-kubelet.io/) provider implementation specific to K3k, which schedules the workloads and other eventually needed resources on the host cluster. This K3k Virtual Kubelet provider handles the reflection of resources and workload execution within the shared host cluster environment.
+
+![Shared Mode](./images/architecture/shared-mode.png)
+
+
+### Networking and Storage
+
+Because of this shared infrastructure, the CNI will be the same one configured in the host cluster. To provide the needed isolation, K3k will leverage Network Policies.
+
+The same goes for the available storage, so the Storage Classes and Volumes are those of the host cluster.
+
+
+### Resource Sharing and Limits
+
+In shared mode, K3k leverages Kubernetes ResourceQuotas and LimitRanges to manage resource sharing and enforce limits.  Since all virtual cluster workloads run within the same namespace on the host cluster, ResourceQuotas are applied to this namespace to limit the total resources consumed by a virtual cluster. LimitRanges are used to set default resource requests and limits for pods, ensuring that workloads have reasonable resource allocations even if they don't explicitly specify them.
+
+Each pod in a virtual cluster is assigned a unique name that incorporates the pod name, namespace, and cluster name. This prevents naming collisions in the shared host cluster namespace.
+
+It's important to understand that ResourceQuotas are applied at the namespace level. This means that all pods within a virtual cluster share the same quota.  While this provides overall limits for the virtual cluster, it also means that resource allocation is dynamic. If one workload isn't using its full resource allocation, other workloads within the *same* virtual cluster can utilize those resources, even if they belong to different deployments or services.
+
+This dynamic sharing can be both a benefit and a challenge.  It allows for efficient resource utilization, but it can also lead to unpredictable performance if workloads have varying resource demands.  Furthermore, this approach makes it difficult to guarantee strict resource isolation between workloads within the same virtual cluster.
+
+GPU resource sharing is an area of ongoing investigation. K3k is actively exploring potential solutions in this area.
+
+
+### Isolation and Security
+
+Isolation between virtual clusters in shared mode relies heavily on Kubernetes Network Policies.  Network Policies define rules that control the network traffic allowed to and from pods.  K3k configures Network Policies to ensure that pods in one virtual cluster cannot communicate with pods in other virtual clusters or with pods in the host cluster itself, providing a strong foundation for network isolation.
+
+While Network Policies offer robust isolation capabilities, it's important to understand their characteristics:
+
+* **CNI Integration:** Network Policies integrate seamlessly with supported CNI plugins.  K3k leverages this integration to enforce network isolation.
+* **Granular Control:** Network Policies provide granular control over network traffic, allowing for fine-tuned security policies.
+* **Scalability:** Network Policies scale well with the number of virtual clusters and applications, ensuring consistent isolation as the environment grows.
+
+K3k also utilizes Kubernetes Pod Security Admission (PSA) to enforce security policies within virtual clusters based on Pod Security Standards (PSS).  PSS define different levels of security for pods, restricting what actions pods can perform.  By configuring PSA to enforce a specific PSS level (e.g., `baseline` or `restricted`) for a virtual cluster, K3k ensures that pods adhere to established security best practices and prevents them from using privileged features or performing potentially dangerous operations.
+
+Key aspects of PSA integration include:
+
+* **Namespace-Level Enforcement:** PSA configuration is applied at the namespace level, providing a consistent security posture for all pods within the virtual cluster.
+* **Standardized Profiles:** PSS offers a set of predefined security profiles aligned with industry best practices, simplifying security configuration and ensuring a baseline level of security.
+
+The shared mode architecture is designed with security in mind.  K3k employs multiple layers of security controls, including Network Policies and PSA, to protect virtual clusters and the host cluster.  While the shared namespace model requires careful configuration and management, these controls provide a robust security foundation for running workloads in a multi-tenant environment.  K3k continuously evaluates and enhances its security mechanisms to address evolving threats and ensure the highest level of protection for its users.
+
+
+## Virtual Mode
+
+The `virtual` mode in K3k deploys fully functional K3s clusters (including both server and agent components) as virtual clusters.  These K3s clusters run as pods within the host cluster.  Each virtual cluster has its own dedicated K3s server and one or more K3s agents acting as worker nodes. This approach provides strong isolation, as each virtual cluster operates independently with its own control plane and worker nodes.  While these virtual clusters run as pods on the host cluster, they function as complete and separate Kubernetes environments.
+
+![Virtual Mode](./images/architecture/virtual-mode.png)
+
+
+### Networking and Storage
+
+Virtual clusters in `virtual` mode each have their own independent networking configuration managed by their respective K3s servers.  Each virtual cluster runs its own CNI plugin, configured within its K3s server, providing complete network isolation from other virtual clusters and the host cluster.  While the virtual cluster networks ultimately operate on top of the host cluster's network infrastructure, the networking configuration and traffic management are entirely separate.
+
+
+### Resource Sharing and Limits
+
+Resource sharing in `virtual` mode is managed by applying resource limits to the pods that make up the virtual cluster (both the K3s server pod and the K3s agent pods).  Each pod is assigned a specific amount of CPU, memory, and other resources.  The workloads running *within* the virtual cluster then utilize these allocated resources.  This means that the virtual cluster as a whole has a defined resource pool determined by the limits on its constituent pods.
+
+This approach provides a clear and direct way to control the resources available to each virtual cluster.  However, it requires careful resource planning to ensure that each virtual cluster has sufficient capacity for its workloads.
+
+
+### Isolation and Security
+
+The `virtual` mode offers strong isolation due to the dedicated K3s clusters deployed for each virtual cluster.  Because each virtual cluster runs its own separate control plane and worker nodes, workloads are effectively isolated from each other and from the host cluster.  This architecture minimizes the risk of one virtual cluster impacting others or the host cluster.
+
+Security in `virtual` mode benefits from the inherent isolation provided by the separate K3s clusters.  However, standard Kubernetes security best practices still apply, and K3k emphasizes a layered security approach.  While the K3s server pods often run with elevated privileges (due to the nature of their function, requiring access to system resources), K3k recommends minimizing these privileges whenever possible and adhering to the principle of least privilege.  This can be achieved by carefully configuring the necessary capabilities instead of relying on full `privileged` mode.  Further information on K3s security best practices can be found in the official K3s documentation: [https://docs.k3s.io/security](https://docs.k3s.io/security) (This link provides general security guidance, including discussions of capabilities and other relevant topics).
+
+Currently security in virtual  mode has a risk of privilege escalation as the server pods run with elevated privileges (due to the nature of their function, requiring access to system resources).
+
+
+## K3k Components
+
+K3k consists of two main components:
+
+* **Controller:** The K3k controller is a core component that runs on the host cluster. It watches for `Cluster` custom resources (CRs) and manages the lifecycle of virtual clusters. When a new `Cluster` CR is created, the controller provisions the necessary resources, including namespaces, K3s server and agent pods, and network configurations, to create the virtual cluster.
+* **CLI:** The K3k CLI provides a command-line interface for interacting with K3k. It allows users to easily create, manage, and access virtual clusters. The CLI simplifies common tasks such as creating `Cluster` CRs, retrieving kubeconfigs for accessing virtual clusters, and performing other management operations.
+
+
+## Comparison and Trade-offs
+
+K3k offers two distinct modes for deploying virtual clusters: `shared` and `virtual`. Each mode has its own strengths and weaknesses, and the best choice depends on the specific needs and priorities of the user. Here's a comparison to help you make an informed decision:
+
+| Feature | Shared Mode | Virtual Mode |
+|---|---|---|
+| **Architecture** | Agentless K3s server with Virtual Kubelet | Full K3s cluster (server and agents) as pods |
+| **Isolation** | Network Policies | Dedicated control plane and worker nodes |
+| **Resource Sharing** | Dynamic, namespace-level ResourceQuotas | Resource limits on virtual cluster pods |
+| **Networking** | Host cluster's CNI | Virtual cluster's own CNI |
+| **Storage** | Host cluster's storage | *Under development* |
+| **Security** | Pod Security Admission (PSA), Network Policies | Inherent isolation, PSA, Network Policies, secure host configuration |
+| **Performance** | Smaller footprint, more efficient due to running directly on the host | Higher overhead due to running full K3s clusters |
+
+**Trade-offs:**
+
+* **Isolation vs. Overhead:** The `shared` mode has lower overhead but weaker isolation, while the `virtual` mode provides stronger isolation but potentially higher overhead due to running full K3s clusters.
+* **Resource Sharing:** The `shared` mode offers dynamic resource sharing within a namespace, which can be efficient but less predictable. The `virtual` mode provides dedicated resources to each virtual cluster, offering more control but requiring careful planning.
+
+**Choosing the right mode:**
+
+* **Choose `shared` mode if:**
+    * You prioritize low overhead and resource efficiency.
+    * You need a simple setup and don't require strong isolation between virtual clusters.
+    * Your workloads don't have strict performance requirements.
+    * Your workloads needs host capacities (GPU)
+* **Choose `virtual` mode if:**
+    * You prioritize strong isolation.
+    * You need dedicated resources and predictable performance for your virtual clusters.
+
+Ultimately, the best choice depends on your specific requirements and priorities. Consider the trade-offs carefully and choose the mode that best aligns with your needs.

docs/crds/Makefile

@@ -0,0 +1,6 @@
+CRD_REF_DOCS_VER := v0.1.0
+CRD_REF_DOCS := go run github.com/elastic/crd-ref-docs@$(CRD_REF_DOCS_VER)
+
+.PHONY: generate
+generate:
+	$(CRD_REF_DOCS) --config=config.yaml --renderer=markdown --source-path=../../pkg/apis/k3k.io/v1alpha1 --output-path=crd-docs.md

docs/crds/config.yaml

@@ -0,0 +1,14 @@
+processor:
+  # RE2 regular expressions describing types that should be excluded from the generated documentation.
+  ignoreTypes:
+    - ClusterSet
+    - ClusterSetList
+
+  # RE2 regular expressions describing type fields that should be excluded from the generated documentation.
+  ignoreFields:
+    - "status$"
+    - "TypeMeta$"
+
+render:
+  # Version of Kubernetes to use when generating links to Kubernetes API documentation.
+  kubernetesVersion: "1.31"

docs/crds/crd-docs.md

@@ -0,0 +1,237 @@
+# API Reference
+
+## Packages
+- [k3k.io/v1alpha1](#k3kiov1alpha1)
+
+
+## k3k.io/v1alpha1
+
+
+### Resource Types
+- [Cluster](#cluster)
+- [ClusterList](#clusterlist)
+
+
+
+#### Addon
+
+
+
+
+
+
+
+_Appears in:_
+- [ClusterSpec](#clusterspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `secretNamespace` _string_ |  |  |  |
+| `secretRef` _string_ |  |  |  |
+
+
+#### Cluster
+
+
+
+
+
+
+
+_Appears in:_
+- [ClusterList](#clusterlist)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `apiVersion` _string_ | `k3k.io/v1alpha1` | | |
+| `kind` _string_ | `Cluster` | | |
+| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. |  |  |
+| `spec` _[ClusterSpec](#clusterspec)_ |  | \{  \} |  |
+
+
+#### ClusterLimit
+
+
+
+
+
+
+
+_Appears in:_
+- [ClusterSpec](#clusterspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `serverLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | ServerLimit is the limits (cpu/mem) that apply to the server nodes |  |  |
+| `workerLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | WorkerLimit is the limits (cpu/mem) that apply to the agent nodes |  |  |
+
+
+#### ClusterList
+
+
+
+
+
+
+
+
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `apiVersion` _string_ | `k3k.io/v1alpha1` | | |
+| `kind` _string_ | `ClusterList` | | |
+| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. |  |  |
+| `items` _[Cluster](#cluster) array_ |  |  |  |
+
+
+#### ClusterMode
+
+_Underlying type:_ _string_
+
+ClusterMode is the possible provisioning mode of a Cluster.
+
+_Validation:_
+- Enum: [shared virtual]
+
+_Appears in:_
+- [ClusterSpec](#clusterspec)
+
+
+
+#### ClusterSpec
+
+
+
+
+
+
+
+_Appears in:_
+- [Cluster](#cluster)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `version` _string_ | Version is a string representing the Kubernetes version to be used by the virtual nodes. |  |  |
+| `servers` _integer_ | Servers is the number of K3s pods to run in server (controlplane) mode. | 1 |  |
+| `agents` _integer_ | Agents is the number of K3s pods to run in agent (worker) mode. | 0 |  |
+| `nodeSelector` _object (keys:string, values:string)_ | NodeSelector is the node selector that will be applied to all server/agent pods.<br />In "shared" mode the node selector will be applied also to the workloads. |  |  |
+| `priorityClass` _string_ | PriorityClass is the priorityClassName that will be applied to all server/agent pods.<br />In "shared" mode the priorityClassName will be applied also to the workloads. |  |  |
+| `clusterLimit` _[ClusterLimit](#clusterlimit)_ | Limit is the limits that apply for the server/worker nodes. |  |  |
+| `tokenSecretRef` _[SecretReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#secretreference-v1-core)_ | TokenSecretRef is Secret reference used as a token join server and worker nodes to the cluster. The controller<br />assumes that the secret has a field "token" in its data, any other fields in the secret will be ignored. |  |  |
+| `clusterCIDR` _string_ | ClusterCIDR is the CIDR range for the pods of the cluster. Defaults to 10.42.0.0/16. |  |  |
+| `serviceCIDR` _string_ | ServiceCIDR is the CIDR range for the services in the cluster. Defaults to 10.43.0.0/16. |  |  |
+| `clusterDNS` _string_ | ClusterDNS is the IP address for the coredns service. Needs to be in the range provided by ServiceCIDR or CoreDNS may not deploy.<br />Defaults to 10.43.0.10. |  |  |
+| `serverArgs` _string array_ | ServerArgs are the ordered key value pairs (e.x. "testArg", "testValue") for the K3s pods running in server mode. |  |  |
+| `agentArgs` _string array_ | AgentArgs are the ordered key value pairs (e.x. "testArg", "testValue") for the K3s pods running in agent mode. |  |  |
+| `tlsSANs` _string array_ | TLSSANs are the subjectAlternativeNames for the certificate the K3s server will use. |  |  |
+| `addons` _[Addon](#addon) array_ | Addons is a list of secrets containing raw YAML which will be deployed in the virtual K3k cluster on startup. |  |  |
+| `mode` _[ClusterMode](#clustermode)_ | Mode is the cluster provisioning mode which can be either "shared" or "virtual". Defaults to "shared" | shared | Enum: [shared virtual] <br /> |
+| `persistence` _[PersistenceConfig](#persistenceconfig)_ | Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data<br />persistence is guaranteed, so restart of a virtual cluster pod may result in data loss without this field. | \{ type:dynamic \} |  |
+| `expose` _[ExposeConfig](#exposeconfig)_ | Expose contains options for exposing the apiserver inside/outside of the cluster. By default, this is only exposed as a<br />clusterIP which is relatively secure, but difficult to access outside of the cluster. |  |  |
+
+
+
+
+#### ExposeConfig
+
+
+
+
+
+
+
+_Appears in:_
+- [ClusterSpec](#clusterspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `ingress` _[IngressConfig](#ingressconfig)_ |  |  |  |
+| `loadbalancer` _[LoadBalancerConfig](#loadbalancerconfig)_ |  |  |  |
+| `nodePort` _[NodePortConfig](#nodeportconfig)_ |  |  |  |
+
+
+#### IngressConfig
+
+
+
+
+
+
+
+_Appears in:_
+- [ExposeConfig](#exposeconfig)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `annotations` _object (keys:string, values:string)_ | Annotations is a key value map that will enrich the Ingress annotations |  |  |
+| `ingressClassName` _string_ |  |  |  |
+
+
+#### LoadBalancerConfig
+
+
+
+
+
+
+
+_Appears in:_
+- [ExposeConfig](#exposeconfig)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `enabled` _boolean_ |  |  |  |
+
+
+#### NodePortConfig
+
+
+
+
+
+
+
+_Appears in:_
+- [ExposeConfig](#exposeconfig)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `serverPort` _integer_ | ServerPort is the port on each node on which the K3s server service is exposed when type is NodePort.<br />If not specified, a port will be allocated (default: 30000-32767) |  |  |
+| `servicePort` _integer_ | ServicePort is the port on each node on which the K3s service is exposed when type is NodePort.<br />If not specified, a port will be allocated (default: 30000-32767) |  |  |
+| `etcdPort` _integer_ | ETCDPort is the port on each node on which the ETCD service is exposed when type is NodePort.<br />If not specified, a port will be allocated (default: 30000-32767) |  |  |
+
+
+#### PersistenceConfig
+
+
+
+
+
+
+
+_Appears in:_
+- [ClusterSpec](#clusterspec)
+- [ClusterStatus](#clusterstatus)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `type` _[PersistenceMode](#persistencemode)_ |  | dynamic |  |
+| `storageClassName` _string_ |  |  |  |
+| `storageRequestSize` _string_ |  |  |  |
+
+
+#### PersistenceMode
+
+_Underlying type:_ _string_
+
+PersistenceMode is the storage mode of a Cluster.
+
+
+
+_Appears in:_
+- [PersistenceConfig](#persistenceconfig)
+
+
+
+
+

docs/development.md

@@ -0,0 +1,15 @@
+# Development
+
+## Tests
+
+To run the tests we use [Ginkgo](https://onsi.github.io/ginkgo/), and [`envtest`](https://book.kubebuilder.io/reference/envtest) for testing the controllers.
+
+Install the required binaries from `envtest` with [`setup-envtest`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/tools/setup-envtest), and then put them in the default path `/usr/local/kubebuilder/bin`:
+
+```
+ENVTEST_BIN=$(setup-envtest use -p path)
+sudo mkdir -p /usr/local/kubebuilder/bin
+sudo cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin
+```
+
+then run `ginkgo run ./...`.

go.mod

@@ -1,8 +1,6 @@
 module github.com/rancher/k3k
 
-go 1.22.0
-
-toolchain go1.22.7
+go 1.23.4
 
 replace (
 	github.com/google/cel-go => github.com/google/cel-go v0.17.7
@@ -14,81 +12,165 @@ replace (
 
 require (
 	github.com/go-logr/zapr v1.3.0
-	github.com/onsi/ginkgo/v2 v2.20.1
+	github.com/onsi/ginkgo/v2 v2.21.0
 	github.com/onsi/gomega v1.36.0
-	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_model v0.6.1
 	github.com/rancher/dynamiclistener v1.27.5
 	github.com/sirupsen/logrus v1.9.3
-	github.com/urfave/cli v1.22.12
+	github.com/stretchr/testify v1.10.0
+	github.com/testcontainers/testcontainers-go v0.35.0
+	github.com/testcontainers/testcontainers-go/modules/k3s v0.35.0
+	github.com/urfave/cli/v2 v2.27.5
 	github.com/virtual-kubelet/virtual-kubelet v1.11.0
-	go.etcd.io/etcd/api/v3 v3.5.14
-	go.etcd.io/etcd/client/v3 v3.5.14
-	go.uber.org/zap v1.26.0
+	go.etcd.io/etcd/api/v3 v3.5.16
+	go.etcd.io/etcd/client/v3 v3.5.16
+	go.uber.org/zap v1.27.0
 	gopkg.in/yaml.v2 v2.4.0
+	helm.sh/helm/v3 v3.14.4
 	k8s.io/api v0.29.11
 	k8s.io/apimachinery v0.29.11
 	k8s.io/apiserver v0.29.11
 	k8s.io/client-go v0.29.11
 	k8s.io/component-base v0.29.11
-	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
 	sigs.k8s.io/controller-runtime v0.17.5
 )
 
 require (
+	dario.cat/mergo v1.0.1 // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
+	github.com/BurntSushi/toml v1.4.0 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.3.0 // indirect
+	github.com/Masterminds/sprig/v3 v3.3.0 // indirect
+	github.com/Masterminds/squirrel v1.5.4 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
-	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
+	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/containerd/containerd v1.7.24 // indirect
+	github.com/containerd/errdefs v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
+	github.com/cyphar/filepath-securejoin v0.3.6 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/cli v25.0.1+incompatible // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v27.1.1+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch v5.9.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
+	github.com/fatih/color v1.13.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/google/cel-go v0.20.1 // indirect
+	github.com/google/btree v1.1.3 // indirect
+	github.com/google/cel-go v0.22.0 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
+	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/gosuri/uitable v0.0.4 // indirect
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/huandu/xstrings v1.5.0 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmoiron/sqlx v1.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
+	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/moby/spdystream v0.4.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/spdystream v0.5.0 // indirect
+	github.com/moby/sys/mountinfo v0.7.2 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/rubenv/sql-migrate v1.7.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/spf13/cast v1.7.0 // indirect
 	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stoewer/go-strcase v1.2.0 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect
+	github.com/stoewer/go-strcase v1.3.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	github.com/xlab/treeprint v1.2.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
@@ -100,30 +182,36 @@ require (
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
-	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
-	golang.org/x/term v0.25.0 // indirect
-	golang.org/x/text v0.19.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.24.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/oauth2 v0.23.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/time v0.7.0 // indirect
+	golang.org/x/tools v0.26.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
 	google.golang.org/grpc v1.65.0 // indirect
 	google.golang.org/protobuf v1.35.1 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.29.2 // indirect
+	k8s.io/apiextensions-apiserver v0.29.11 // indirect
+	k8s.io/cli-runtime v0.29.11 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
-	k8s.io/kms v0.31.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
-	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect
-	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	k8s.io/kms v0.29.11 // indirect
+	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
+	k8s.io/kubectl v0.29.11 // indirect
+	oras.land/oras-go v1.2.5 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/kustomize/api v0.18.0 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.3 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )

k3k-kubelet/config.go

@@ -11,13 +11,14 @@ import (
 type config struct {
 	ClusterName       string `yaml:"clusterName,omitempty"`
 	ClusterNamespace  string `yaml:"clusterNamespace,omitempty"`
-	NodeName          string `yaml:"nodeName,omitempty"`
+	ServiceName       string `yaml:"serviceName,omitempty"`
 	Token             string `yaml:"token,omitempty"`
 	AgentHostname     string `yaml:"agentHostname,omitempty"`
 	HostConfigPath    string `yaml:"hostConfigPath,omitempty"`
 	VirtualConfigPath string `yaml:"virtualConfigPath,omitempty"`
 	KubeletPort       string `yaml:"kubeletPort,omitempty"`
 	ServerIP          string `yaml:"serverIP,omitempty"`
+	Version           string `yaml:"version,omitempty"`
 }
 
 func (c *config) unmarshalYAML(data []byte) error {
@@ -45,8 +46,8 @@ func (c *config) unmarshalYAML(data []byte) error {
 	if c.AgentHostname == "" {
 		c.AgentHostname = conf.AgentHostname
 	}
-	if c.NodeName == "" {
-		c.NodeName = conf.NodeName
+	if c.ServiceName == "" {
+		c.ServiceName = conf.ServiceName
 	}
 	if c.Token == "" {
 		c.Token = conf.Token
@@ -54,6 +55,9 @@ func (c *config) unmarshalYAML(data []byte) error {
 	if c.ServerIP == "" {
 		c.ServerIP = conf.ServerIP
 	}
+	if c.Version == "" {
+		c.Version = conf.Version
+	}
 	return nil
 }
 

k3k-kubelet/controller/handler.go

@@ -26,9 +26,9 @@ type ControllerHandler struct {
 	HostClient client.Client
 	// VirtualClient is the client used to communicate with the virtual cluster
 	VirtualClient client.Client
-	// Translater is the translater that will be used to adjust objects before they
+	// Translator is the translator that will be used to adjust objects before they
 	// are made on the host cluster
-	Translater translate.ToHostTranslater
+	Translator translate.ToHostTranslator
 	// Logger is the logger that the controller will use to log errors
 	Logger *k3klog.Logger
 	// controllers are the controllers which are currently running
@@ -65,7 +65,11 @@ func (c *ControllerHandler) AddResource(ctx context.Context, obj client.Object)
 			TranslateFunc: func(s *v1.Secret) (*v1.Secret, error) {
 				// note that this doesn't do any type safety - fix this
 				// when generics work
-				c.Translater.TranslateTo(s)
+				c.Translator.TranslateTo(s)
+				// Remove service-account-token types when synced to the host
+				if s.Type == v1.SecretTypeServiceAccountToken {
+					s.Type = v1.SecretTypeOpaque
+				}
 				return s, nil
 			},
 			Logger: c.Logger,
@@ -76,7 +80,7 @@ func (c *ControllerHandler) AddResource(ctx context.Context, obj client.Object)
 			VirtualClient: c.VirtualClient,
 			// TODO: Need actual function
 			TranslateFunc: func(s *v1.ConfigMap) (*v1.ConfigMap, error) {
-				c.Translater.TranslateTo(s)
+				c.Translator.TranslateTo(s)
 				return s, nil
 			},
 			Logger: c.Logger,
@@ -109,7 +113,7 @@ func (c *ControllerHandler) RemoveResource(ctx context.Context, obj client.Objec
 	ctrl, ok := c.controllers[obj.GetObjectKind().GroupVersionKind()]
 	c.RUnlock()
 	if !ok {
-		return fmt.Errorf("no controller found for gvk" + obj.GetObjectKind().GroupVersionKind().String())
+		return fmt.Errorf("no controller found for gvk %s", obj.GetObjectKind().GroupVersionKind())
 	}
 	return ctrl.RemoveResource(ctx, obj.GetNamespace(), obj.GetName())
 }

k3k-kubelet/controller/persistentvolumeclaims.go

@@ -31,12 +31,12 @@ type PVCReconciler struct {
 	Scheme           *runtime.Scheme
 	HostScheme       *runtime.Scheme
 	logger           *log.Logger
-	Translater       translate.ToHostTranslater
+	Translator       translate.ToHostTranslator
 }
 
 // AddPVCSyncer adds persistentvolumeclaims syncer controller to k3k-kubelet
 func AddPVCSyncer(ctx context.Context, virtMgr, hostMgr manager.Manager, clusterName, clusterNamespace string, logger *log.Logger) error {
-	translater := translate.ToHostTranslater{
+	translator := translate.ToHostTranslator{
 		ClusterName:      clusterName,
 		ClusterNamespace: clusterNamespace,
 	}
@@ -47,7 +47,7 @@ func AddPVCSyncer(ctx context.Context, virtMgr, hostMgr manager.Manager, cluster
 		Scheme:           virtMgr.GetScheme(),
 		HostScheme:       hostMgr.GetScheme(),
 		logger:           logger.Named(pvcController),
-		Translater:       translater,
+		Translator:       translator,
 		clusterName:      clusterName,
 		clusterNamespace: clusterNamespace,
 	}
@@ -116,6 +116,6 @@ func (r *PVCReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r
 
 func (r *PVCReconciler) pvc(obj *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim {
 	hostPVC := obj.DeepCopy()
-	r.Translater.TranslateTo(hostPVC)
+	r.Translator.TranslateTo(hostPVC)
 	return hostPVC
 }

k3k-kubelet/controller/service.go

@@ -33,12 +33,12 @@ type ServiceReconciler struct {
 	Scheme           *runtime.Scheme
 	HostScheme       *runtime.Scheme
 	logger           *log.Logger
-	Translater       translate.ToHostTranslater
+	Translator       translate.ToHostTranslator
 }
 
 // AddServiceSyncer adds service syncer controller to the manager of the virtual cluster
 func AddServiceSyncer(ctx context.Context, virtMgr, hostMgr manager.Manager, clusterName, clusterNamespace string, logger *log.Logger) error {
-	translater := translate.ToHostTranslater{
+	translator := translate.ToHostTranslator{
 		ClusterName:      clusterName,
 		ClusterNamespace: clusterNamespace,
 	}
@@ -49,7 +49,7 @@ func AddServiceSyncer(ctx context.Context, virtMgr, hostMgr manager.Manager, clu
 		Scheme:           virtMgr.GetScheme(),
 		HostScheme:       hostMgr.GetScheme(),
 		logger:           logger.Named(serviceSyncerController),
-		Translater:       translater,
+		Translator:       translator,
 		clusterName:      clusterName,
 		clusterNamespace: clusterNamespace,
 	}
@@ -120,7 +120,7 @@ func (s *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 
 func (s *ServiceReconciler) service(obj *v1.Service) *v1.Service {
 	hostService := obj.DeepCopy()
-	s.Translater.TranslateTo(hostService)
+	s.Translator.TranslateTo(hostService)
 	// don't sync finalizers to the host
 	return hostService
 }

k3k-kubelet/controller/webhook/pod.go

@@ -4,11 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strconv"
+	"strings"
 
 	"github.com/rancher/k3k/pkg/controller/cluster/agent"
 	"github.com/rancher/k3k/pkg/log"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -19,16 +22,18 @@ import (
 )
 
 const (
-	webhookName    = "nodename.podmutator.k3k.io"
+	webhookName    = "podmutator.k3k.io"
 	webhookTimeout = int32(10)
 	webhookPort    = "9443"
 	webhookPath    = "/mutate--v1-pod"
+	FieldpathField = "k3k.io/fieldpath"
 )
 
 type webhookHandler struct {
 	client           ctrlruntimeclient.Client
 	scheme           *runtime.Scheme
 	nodeName         string
+	serviceName      string
 	clusterName      string
 	clusterNamespace string
 	logger           *log.Logger
@@ -36,11 +41,13 @@ type webhookHandler struct {
 
 // AddPodMutatorWebhook will add a mutator webhook to the virtual cluster to
 // modify the nodeName of the created pods with the name of the virtual kubelet node name
-func AddPodMutatorWebhook(ctx context.Context, mgr manager.Manager, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, nodeName string, logger *log.Logger) error {
+// as well as remove any status fields of the downward apis env fields
+func AddPodMutatorWebhook(ctx context.Context, mgr manager.Manager, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, nodeName, serviceName string, logger *log.Logger) error {
 	handler := webhookHandler{
 		client:           mgr.GetClient(),
 		scheme:           mgr.GetScheme(),
 		logger:           logger,
+		serviceName:      serviceName,
 		clusterName:      clusterName,
 		clusterNamespace: clusterNamespace,
 		nodeName:         nodeName,
@@ -52,7 +59,9 @@ func AddPodMutatorWebhook(ctx context.Context, mgr manager.Manager, hostClient c
 		return err
 	}
 	if err := handler.client.Create(ctx, config); err != nil {
-		return err
+		if !apierrors.IsAlreadyExists(err) {
+			return err
+		}
 	}
 	// register webhook with the manager
 	return ctrl.NewWebhookManagedBy(mgr).For(&v1.Pod{}).WithDefaulter(&handler).Complete()
@@ -63,10 +72,28 @@ func (w *webhookHandler) Default(ctx context.Context, obj runtime.Object) error
 	if !ok {
 		return fmt.Errorf("invalid request: object was type %t not cluster", obj)
 	}
-	w.logger.Infow("recieved request", "Pod", pod.Name, "Namespace", pod.Namespace)
+	w.logger.Infow("mutator webhook request", "Pod", pod.Name, "Namespace", pod.Namespace)
 	if pod.Spec.NodeName == "" {
 		pod.Spec.NodeName = w.nodeName
 	}
+	// look for status.* fields in the env
+	if pod.Annotations == nil {
+		pod.Annotations = make(map[string]string)
+	}
+	for i, container := range pod.Spec.Containers {
+		for j, env := range container.Env {
+			if env.ValueFrom == nil || env.ValueFrom.FieldRef == nil {
+				continue
+			}
+
+			fieldPath := env.ValueFrom.FieldRef.FieldPath
+			if strings.Contains(fieldPath, "status.") {
+				annotationKey := fmt.Sprintf("%s_%d_%s", FieldpathField, i, env.Name)
+				pod.Annotations[annotationKey] = fieldPath
+				pod.Spec.Containers[i].Env = removeEnv(pod.Spec.Containers[i].Env, j)
+			}
+		}
+	}
 	return nil
 }
 
@@ -82,7 +109,7 @@ func (w *webhookHandler) configuration(ctx context.Context, hostClient ctrlrunti
 	if !ok {
 		return nil, errors.New("webhook CABundle does not exist in secret")
 	}
-	webhookURL := "https://" + w.nodeName + ":" + webhookPort + webhookPath
+	webhookURL := "https://" + w.serviceName + ":" + webhookPort + webhookPath
 	return &admissionregistrationv1.MutatingWebhookConfiguration{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "admissionregistration.k8s.io/v1",
@@ -118,3 +145,22 @@ func (w *webhookHandler) configuration(ctx context.Context, hostClient ctrlrunti
 		},
 	}, nil
 }
+
+func removeEnv(envs []v1.EnvVar, i int) []v1.EnvVar {
+	envs[i] = envs[len(envs)-1]
+	return envs[:len(envs)-1]
+}
+
+func ParseFieldPathAnnotationKey(annotationKey string) (int, string, error) {
+	s := strings.SplitN(annotationKey, "_", 3)
+	if len(s) != 3 {
+		return -1, "", errors.New("fieldpath annotation is not set correctly")
+	}
+	containerIndex, err := strconv.Atoi(s[1])
+	if err != nil {
+		return -1, "", err
+	}
+	envName := s[2]
+
+	return containerIndex, envName, nil
+}

k3k-kubelet/kubelet.go

@@ -127,7 +127,7 @@ func newKubelet(ctx context.Context, c *config, logger *k3klog.Logger) (*kubelet
 		return nil, errors.New("unable to create controller-runtime mgr for virtual cluster: " + err.Error())
 	}
 	logger.Info("adding pod mutator webhook")
-	if err := k3kwebhook.AddPodMutatorWebhook(ctx, virtualMgr, hostClient, c.ClusterName, c.ClusterNamespace, c.NodeName, logger); err != nil {
+	if err := k3kwebhook.AddPodMutatorWebhook(ctx, virtualMgr, hostClient, c.ClusterName, c.ClusterNamespace, c.AgentHostname, c.ServiceName, logger); err != nil {
 		return nil, errors.New("unable to add pod mutator webhook for virtual cluster: " + err.Error())
 	}
 
@@ -141,7 +141,7 @@ func newKubelet(ctx context.Context, c *config, logger *k3klog.Logger) (*kubelet
 		return nil, errors.New("failed to add pvc syncer controller: " + err.Error())
 	}
 
-	clusterIP, err := clusterIP(ctx, c.AgentHostname, c.ClusterNamespace, hostClient)
+	clusterIP, err := clusterIP(ctx, c.ServiceName, c.ClusterNamespace, hostClient)
 	if err != nil {
 		return nil, errors.New("failed to extract the clusterIP for the server service: " + err.Error())
 	}
@@ -161,7 +161,7 @@ func newKubelet(ctx context.Context, c *config, logger *k3klog.Logger) (*kubelet
 	return &kubelet{
 		virtualCluster: virtualCluster,
 
-		name:       c.NodeName,
+		name:       c.AgentHostname,
 		hostConfig: hostConfig,
 		hostClient: hostClient,
 		virtConfig: virtConfig,
@@ -184,8 +184,8 @@ func clusterIP(ctx context.Context, serviceName, clusterNamespace string, hostCl
 	return service.Spec.ClusterIP, nil
 }
 
-func (k *kubelet) registerNode(ctx context.Context, agentIP, srvPort, namespace, name, hostname, serverIP, dnsIP string) error {
-	providerFunc := k.newProviderFunc(namespace, name, hostname, agentIP, serverIP, dnsIP)
+func (k *kubelet) registerNode(ctx context.Context, agentIP, srvPort, namespace, name, hostname, serverIP, dnsIP, version string) error {
+	providerFunc := k.newProviderFunc(namespace, name, hostname, agentIP, serverIP, dnsIP, version)
 	nodeOpts := k.nodeOpts(ctx, srvPort, namespace, name, hostname, agentIP)
 
 	var err error
@@ -232,14 +232,14 @@ func (k *kubelet) start(ctx context.Context) {
 	k.logger.Info("node exited successfully")
 }
 
-func (k *kubelet) newProviderFunc(namespace, name, hostname, agentIP, serverIP, dnsIP string) nodeutil.NewProviderFunc {
+func (k *kubelet) newProviderFunc(namespace, name, hostname, agentIP, serverIP, dnsIP, version string) nodeutil.NewProviderFunc {
 	return func(pc nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) {
 		utilProvider, err := provider.New(*k.hostConfig, k.hostMgr, k.virtualMgr, k.logger, namespace, name, serverIP, dnsIP)
 		if err != nil {
 			return nil, nil, errors.New("unable to make nodeutil provider: " + err.Error())
 		}
 
-		provider.ConfigureNode(k.logger, pc.Node, hostname, k.port, agentIP, utilProvider.CoreClient, utilProvider.VirtualClient, k.virtualCluster)
+		provider.ConfigureNode(k.logger, pc.Node, hostname, k.port, agentIP, utilProvider.CoreClient, utilProvider.VirtualClient, k.virtualCluster, version)
 
 		return utilProvider, &provider.Node{}, nil
 	}

k3k-kubelet/main.go

@@ -7,7 +7,7 @@ import (
 	"github.com/go-logr/zapr"
 	"github.com/rancher/k3k/pkg/log"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 	"go.uber.org/zap"
 	ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log"
 )
@@ -24,67 +24,79 @@ func main() {
 	app.Name = "k3k-kubelet"
 	app.Usage = "virtual kubelet implementation k3k"
 	app.Flags = []cli.Flag{
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "cluster-name",
 			Usage:       "Name of the k3k cluster",
 			Destination: &cfg.ClusterName,
-			EnvVar:      "CLUSTER_NAME",
+			EnvVars:     []string{"CLUSTER_NAME"},
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "cluster-namespace",
 			Usage:       "Namespace of the k3k cluster",
 			Destination: &cfg.ClusterNamespace,
-			EnvVar:      "CLUSTER_NAMESPACE",
+			EnvVars:     []string{"CLUSTER_NAMESPACE"},
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "cluster-token",
 			Usage:       "K3S token of the k3k cluster",
 			Destination: &cfg.Token,
-			EnvVar:      "CLUSTER_TOKEN",
+			EnvVars:     []string{"CLUSTER_TOKEN"},
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "host-config-path",
 			Usage:       "Path to the host kubeconfig, if empty then virtual-kubelet will use incluster config",
 			Destination: &cfg.HostConfigPath,
-			EnvVar:      "HOST_KUBECONFIG",
+			EnvVars:     []string{"HOST_KUBECONFIG"},
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "virtual-config-path",
 			Usage:       "Path to the k3k cluster kubeconfig, if empty then virtual-kubelet will create its own config from k3k cluster",
 			Destination: &cfg.VirtualConfigPath,
-			EnvVar:      "CLUSTER_NAME",
+			EnvVars:     []string{"CLUSTER_NAME"},
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "kubelet-port",
 			Usage:       "kubelet API port number",
 			Destination: &cfg.KubeletPort,
-			EnvVar:      "SERVER_PORT",
+			EnvVars:     []string{"SERVER_PORT"},
 			Value:       "10250",
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
+			Name:        "service-name",
+			Usage:       "The service name deployed by the k3k controller",
+			Destination: &cfg.ServiceName,
+			EnvVars:     []string{"SERVICE_NAME"},
+		},
+		&cli.StringFlag{
 			Name:        "agent-hostname",
 			Usage:       "Agent Hostname used for TLS SAN for the kubelet server",
 			Destination: &cfg.AgentHostname,
-			EnvVar:      "AGENT_HOSTNAME",
+			EnvVars:     []string{"AGENT_HOSTNAME"},
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "server-ip",
 			Usage:       "Server IP used for registering the virtual kubelet to the cluster",
 			Destination: &cfg.ServerIP,
-			EnvVar:      "SERVER_IP",
+			EnvVars:     []string{"SERVER_IP"},
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
+			Name:        "version",
+			Usage:       "Version of kubernetes server",
+			Destination: &cfg.Version,
+			EnvVars:     []string{"VERSION"},
+		},
+		&cli.StringFlag{
 			Name:        "config",
 			Usage:       "Path to k3k-kubelet config file",
 			Destination: &configFile,
-			EnvVar:      "CONFIG_FILE",
+			EnvVars:     []string{"CONFIG_FILE"},
 			Value:       "/etc/rancher/k3k/config.yaml",
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "Enable debug logging",
 			Destination: &debug,
-			EnvVar:      "DEBUG",
+			EnvVars:     []string{"DEBUG"},
 		},
 	}
 	app.Before = func(clx *cli.Context) error {
@@ -98,7 +110,7 @@ func main() {
 	}
 }
 
-func run(clx *cli.Context) {
+func run(clx *cli.Context) error {
 	ctx := context.Background()
 	if err := cfg.parse(configFile); err != nil {
 		logger.Fatalw("failed to parse config file", "path", configFile, zap.Error(err))
@@ -112,9 +124,11 @@ func run(clx *cli.Context) {
 		logger.Fatalw("failed to create new virtual kubelet instance", zap.Error(err))
 	}
 
-	if err := k.registerNode(ctx, k.agentIP, cfg.KubeletPort, cfg.ClusterNamespace, cfg.ClusterName, cfg.AgentHostname, cfg.ServerIP, k.dnsIP); err != nil {
+	if err := k.registerNode(ctx, k.agentIP, cfg.KubeletPort, cfg.ClusterNamespace, cfg.ClusterName, cfg.AgentHostname, cfg.ServerIP, k.dnsIP, cfg.Version); err != nil {
 		logger.Fatalw("failed to register new node", zap.Error(err))
 	}
 
 	k.start(ctx)
+
+	return nil
 }

k3k-kubelet/provider/configure.go

@@ -15,7 +15,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-func ConfigureNode(logger *k3klog.Logger, node *v1.Node, hostname string, servicePort int, ip string, coreClient typedv1.CoreV1Interface, virtualClient client.Client, virtualCluster v1alpha1.Cluster) {
+func ConfigureNode(logger *k3klog.Logger, node *v1.Node, hostname string, servicePort int, ip string, coreClient typedv1.CoreV1Interface, virtualClient client.Client, virtualCluster v1alpha1.Cluster, version string) {
 	node.Status.Conditions = nodeConditions()
 	node.Status.DaemonEndpoints.KubeletEndpoint.Port = int32(servicePort)
 	node.Status.Addresses = []v1.NodeAddress{
@@ -32,6 +32,10 @@ func ConfigureNode(logger *k3klog.Logger, node *v1.Node, hostname string, servic
 	node.Labels["node.kubernetes.io/exclude-from-external-load-balancers"] = "true"
 	node.Labels["kubernetes.io/os"] = "linux"
 
+	// configure versions
+	node.Status.NodeInfo.KubeletVersion = version
+	node.Status.NodeInfo.KubeProxyVersion = version
+
 	updateNodeCapacityInterval := 10 * time.Second
 	ticker := time.NewTicker(updateNodeCapacityInterval)
 

k3k-kubelet/provider/provider.go

@@ -8,16 +8,18 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
 
-	"github.com/pkg/errors"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/rancher/k3k/k3k-kubelet/controller"
+	"github.com/rancher/k3k/k3k-kubelet/controller/webhook"
 	"github.com/rancher/k3k/k3k-kubelet/provider/collectors"
 	"github.com/rancher/k3k/k3k-kubelet/translate"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	k3klog "github.com/rancher/k3k/pkg/log"
 	"github.com/virtual-kubelet/virtual-kubelet/node/api"
 	"github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
+	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -26,9 +28,12 @@ import (
 	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/scheme"
 	cv1 "k8s.io/client-go/kubernetes/typed/core/v1"
 
+	"errors"
+
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/portforward"
 	"k8s.io/client-go/tools/remotecommand"
@@ -38,11 +43,14 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 )
 
+// check at compile time if the Provider implements the nodeutil.Provider interface
+var _ nodeutil.Provider = (*Provider)(nil)
+
 // Provider implements nodetuil.Provider from virtual Kubelet.
 // TODO: Implement NotifyPods and the required usage so that this can be an async provider
 type Provider struct {
 	Handler          controller.ControllerHandler
-	Translater       translate.ToHostTranslater
+	Translator       translate.ToHostTranslator
 	HostClient       client.Client
 	VirtualClient    client.Client
 	ClientConfig     rest.Config
@@ -54,13 +62,17 @@ type Provider struct {
 	logger           *k3klog.Logger
 }
 
+var (
+	ErrRetryTimeout = errors.New("provider timed out")
+)
+
 func New(hostConfig rest.Config, hostMgr, virtualMgr manager.Manager, logger *k3klog.Logger, namespace, name, serverIP, dnsIP string) (*Provider, error) {
 	coreClient, err := cv1.NewForConfig(&hostConfig)
 	if err != nil {
 		return nil, err
 	}
 
-	translater := translate.ToHostTranslater{
+	translator := translate.ToHostTranslator{
 		ClusterName:      name,
 		ClusterNamespace: namespace,
 	}
@@ -71,12 +83,12 @@ func New(hostConfig rest.Config, hostMgr, virtualMgr manager.Manager, logger *k3
 			Scheme:        *virtualMgr.GetScheme(),
 			HostClient:    hostMgr.GetClient(),
 			VirtualClient: virtualMgr.GetClient(),
-			Translater:    translater,
+			Translator:    translator,
 			Logger:        logger,
 		},
 		HostClient:       hostMgr.GetClient(),
 		VirtualClient:    virtualMgr.GetClient(),
-		Translater:       translater,
+		Translator:       translator,
 		ClientConfig:     hostConfig,
 		CoreClient:       coreClient,
 		ClusterNamespace: namespace,
@@ -91,7 +103,7 @@ func New(hostConfig rest.Config, hostMgr, virtualMgr manager.Manager, logger *k3
 
 // GetContainerLogs retrieves the logs of a container by name from the provider.
 func (p *Provider) GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error) {
-	hostPodName := p.Translater.TranslateName(namespace, podName)
+	hostPodName := p.Translator.TranslateName(namespace, podName)
 	options := corev1.PodLogOptions{
 		Container:  containerName,
 		Timestamps: opts.Timestamps,
@@ -122,7 +134,7 @@ func (p *Provider) GetContainerLogs(ctx context.Context, namespace, podName, con
 // RunInContainer executes a command in a container in the pod, copying data
 // between in/out/err and the container's stdin/stdout/stderr.
 func (p *Provider) RunInContainer(ctx context.Context, namespace, podName, containerName string, cmd []string, attach api.AttachIO) error {
-	hostPodName := p.Translater.TranslateName(namespace, podName)
+	hostPodName := p.Translator.TranslateName(namespace, podName)
 	req := p.CoreClient.RESTClient().Post().
 		Resource("pods").
 		Name(hostPodName).
@@ -154,7 +166,7 @@ func (p *Provider) RunInContainer(ctx context.Context, namespace, podName, conta
 // AttachToContainer attaches to the executing process of a container in the pod, copying data
 // between in/out/err and the container's stdin/stdout/stderr.
 func (p *Provider) AttachToContainer(ctx context.Context, namespace, podName, containerName string, attach api.AttachIO) error {
-	hostPodName := p.Translater.TranslateName(namespace, podName)
+	hostPodName := p.Translator.TranslateName(namespace, podName)
 	req := p.CoreClient.RESTClient().Post().
 		Resource("pods").
 		Name(hostPodName).
@@ -229,7 +241,7 @@ func (p *Provider) GetStatsSummary(ctx context.Context) (*statsv1alpha1.Summary,
 
 	podsNameMap := make(map[string]*v1.Pod)
 	for _, pod := range pods {
-		hostPodName := p.Translater.TranslateName(pod.Namespace, pod.Name)
+		hostPodName := p.Translator.TranslateName(pod.Namespace, pod.Name)
 		podsNameMap[hostPodName] = pod
 	}
 
@@ -262,7 +274,7 @@ func (p *Provider) GetStatsSummary(ctx context.Context) (*statsv1alpha1.Summary,
 func (p *Provider) GetMetricsResource(ctx context.Context) ([]*dto.MetricFamily, error) {
 	statsSummary, err := p.GetStatsSummary(ctx)
 	if err != nil {
-		return nil, errors.Wrapf(err, "error fetching MetricsResource")
+		return nil, errors.Join(err, errors.New("error fetching MetricsResource"))
 	}
 
 	registry := compbasemetrics.NewKubeRegistry()
@@ -270,14 +282,14 @@ func (p *Provider) GetMetricsResource(ctx context.Context) ([]*dto.MetricFamily,
 
 	metricFamily, err := registry.Gather()
 	if err != nil {
-		return nil, errors.Wrapf(err, "error gathering metrics from collector")
+		return nil, errors.Join(err, errors.New("error gathering metrics from collector"))
 	}
 	return metricFamily, nil
 }
 
 // PortForward forwards a local port to a port on the pod
 func (p *Provider) PortForward(ctx context.Context, namespace, pod string, port int32, stream io.ReadWriteCloser) error {
-	hostPodName := p.Translater.TranslateName(namespace, pod)
+	hostPodName := p.Translator.TranslateName(namespace, pod)
 	req := p.CoreClient.RESTClient().Post().
 		Resource("pods").
 		Name(hostPodName).
@@ -306,10 +318,15 @@ func (p *Provider) PortForward(ctx context.Context, namespace, pod string, port
 	return fw.ForwardPorts()
 }
 
-// CreatePod takes a Kubernetes Pod and deploys it within the provider.
+// CreatePod executes createPod with retry
 func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
+	return p.withRetry(ctx, p.createPod, pod)
+}
+
+// createPod takes a Kubernetes Pod and deploys it within the provider.
+func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error {
 	tPod := pod.DeepCopy()
-	p.Translater.TranslateTo(tPod)
+	p.Translator.TranslateTo(tPod)
 
 	// get Cluster definition
 	clusterKey := types.NamespacedName{
@@ -330,6 +347,11 @@ func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
 
 	tPod.Spec.NodeSelector = cluster.Spec.NodeSelector
 
+	// setting the hostname for the pod if its not set
+	if pod.Spec.Hostname == "" {
+		tPod.Spec.Hostname = pod.Name
+	}
+
 	// if the priorityCluss for the virtual cluster is set then override the provided value
 	// Note: the core-dns and local-path-provisioner pod are scheduled by k3s with the
 	// 'system-cluster-critical' and 'system-node-critical' default priority classes.
@@ -338,6 +360,10 @@ func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
 		tPod.Spec.Priority = nil
 	}
 
+	// fieldpath annotations
+	if err := p.configureFieldPathEnv(pod, tPod); err != nil {
+		return fmt.Errorf("unable to fetch fieldpath annotations for pod %s/%s: %w", pod.Namespace, pod.Name, err)
+	}
 	// volumes will often refer to resources in the virtual cluster, but instead need to refer to the sync'd
 	// host cluster version
 	if err := p.transformVolumes(ctx, pod.Namespace, tPod.Spec.Volumes); err != nil {
@@ -348,13 +374,35 @@ func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
 		return fmt.Errorf("unable to transform tokens for pod %s/%s: %w", pod.Namespace, pod.Name, err)
 	}
 	// inject networking information to the pod including the virtual cluster controlplane endpoint
-	p.configureNetworking(pod.Name, pod.Namespace, tPod)
+	p.configureNetworking(pod.Name, pod.Namespace, tPod, p.serverIP)
 
 	p.logger.Infow("Creating pod", "Host Namespace", tPod.Namespace, "Host Name", tPod.Name,
 		"Virtual Namespace", pod.Namespace, "Virtual Name", "env", pod.Name, pod.Spec.Containers[0].Env)
 	return p.HostClient.Create(ctx, tPod)
 }
 
+// withRetry retries passed function with interval and timeout
+func (p *Provider) withRetry(ctx context.Context, f func(context.Context, *v1.Pod) error, pod *v1.Pod) error {
+	const (
+		interval = 2 * time.Second
+		timeout  = 10 * time.Second
+	)
+	var allErrors error
+	// retryFn will retry until the operation succeed, or the timeout occurs
+	retryFn := func(ctx context.Context) (bool, error) {
+		if lastErr := f(ctx, pod); lastErr != nil {
+			// log that the retry failed?
+			allErrors = errors.Join(allErrors, lastErr)
+			return false, nil
+		}
+		return true, nil
+	}
+	if err := wait.PollUntilContextTimeout(ctx, interval, timeout, true, retryFn); err != nil {
+		return errors.Join(allErrors, ErrRetryTimeout)
+	}
+	return nil
+}
+
 // transformVolumes changes the volumes to the representation in the host cluster. Will return an error
 // if one/more volumes couldn't be transformed
 func (p *Provider) transformVolumes(ctx context.Context, podNamespace string, volumes []corev1.Volume) error {
@@ -371,7 +419,7 @@ func (p *Provider) transformVolumes(ctx context.Context, podNamespace string, vo
 			if err := p.syncConfigmap(ctx, podNamespace, volume.ConfigMap.Name, optional); err != nil {
 				return fmt.Errorf("unable to sync configmap volume %s: %w", volume.Name, err)
 			}
-			volume.ConfigMap.Name = p.Translater.TranslateName(podNamespace, volume.ConfigMap.Name)
+			volume.ConfigMap.Name = p.Translator.TranslateName(podNamespace, volume.ConfigMap.Name)
 		} else if volume.Secret != nil {
 			if volume.Secret.Optional != nil {
 				optional = *volume.Secret.Optional
@@ -379,7 +427,7 @@ func (p *Provider) transformVolumes(ctx context.Context, podNamespace string, vo
 			if err := p.syncSecret(ctx, podNamespace, volume.Secret.SecretName, optional); err != nil {
 				return fmt.Errorf("unable to sync secret volume %s: %w", volume.Name, err)
 			}
-			volume.Secret.SecretName = p.Translater.TranslateName(podNamespace, volume.Secret.SecretName)
+			volume.Secret.SecretName = p.Translator.TranslateName(podNamespace, volume.Secret.SecretName)
 		} else if volume.Projected != nil {
 			for _, source := range volume.Projected.Sources {
 				if source.ConfigMap != nil {
@@ -390,7 +438,7 @@ func (p *Provider) transformVolumes(ctx context.Context, podNamespace string, vo
 					if err := p.syncConfigmap(ctx, podNamespace, configMapName, optional); err != nil {
 						return fmt.Errorf("unable to sync projected configmap %s: %w", configMapName, err)
 					}
-					source.ConfigMap.Name = p.Translater.TranslateName(podNamespace, configMapName)
+					source.ConfigMap.Name = p.Translator.TranslateName(podNamespace, configMapName)
 				} else if source.Secret != nil {
 					if source.Secret.Optional != nil {
 						optional = *source.Secret.Optional
@@ -402,12 +450,22 @@ func (p *Provider) transformVolumes(ctx context.Context, podNamespace string, vo
 				}
 			}
 		} else if volume.PersistentVolumeClaim != nil {
-			volume.PersistentVolumeClaim.ClaimName = p.Translater.TranslateName(podNamespace, volume.PersistentVolumeClaim.ClaimName)
+			volume.PersistentVolumeClaim.ClaimName = p.Translator.TranslateName(podNamespace, volume.PersistentVolumeClaim.ClaimName)
+		} else if volume.DownwardAPI != nil {
+			for _, downwardAPI := range volume.DownwardAPI.Items {
+				if downwardAPI.FieldRef.FieldPath == translate.MetadataNameField {
+					downwardAPI.FieldRef.FieldPath = fmt.Sprintf("metadata.annotations['%s']", translate.ResourceNameAnnotation)
+				}
+				if downwardAPI.FieldRef.FieldPath == translate.MetadataNamespaceField {
+					downwardAPI.FieldRef.FieldPath = fmt.Sprintf("metadata.annotations['%s']", translate.ResourceNamespaceAnnotation)
+				}
+			}
 		}
 	}
 	return nil
 }
 
+// syncConfigmap will add the configmap object to the queue of the syncer controller to be synced to the host cluster
 func (p *Provider) syncConfigmap(ctx context.Context, podNamespace string, configMapName string, optional bool) error {
 	var configMap corev1.ConfigMap
 	nsName := types.NamespacedName{
@@ -429,7 +487,9 @@ func (p *Provider) syncConfigmap(ctx context.Context, podNamespace string, confi
 	return nil
 }
 
+// syncSecret will add the secret object to the queue of the syncer controller to be synced to the host cluster
 func (p *Provider) syncSecret(ctx context.Context, podNamespace string, secretName string, optional bool) error {
+	p.logger.Infow("Syncing secret", "Name", secretName, "Namespace", podNamespace, "optional", optional)
 	var secret corev1.Secret
 	nsName := types.NamespacedName{
 		Namespace: podNamespace,
@@ -444,46 +504,103 @@ func (p *Provider) syncSecret(ctx context.Context, podNamespace string, secretNa
 	}
 	err = p.Handler.AddResource(ctx, &secret)
 	if err != nil {
-		return fmt.Errorf("unable to add configmap to sync %s/%s: %w", nsName.Namespace, nsName.Name, err)
+		return fmt.Errorf("unable to add secret to sync %s/%s: %w", nsName.Namespace, nsName.Name, err)
 	}
 	return nil
 }
 
-// UpdatePod takes a Kubernetes Pod and updates it within the provider.
+// UpdatePod executes updatePod with retry
 func (p *Provider) UpdatePod(ctx context.Context, pod *corev1.Pod) error {
-	hostName := p.Translater.TranslateName(pod.Namespace, pod.Name)
-	currentPod, err := p.GetPod(ctx, p.ClusterNamespace, hostName)
-	if err != nil {
-		return fmt.Errorf("unable to get current pod for update: %w", err)
-	}
-	tPod := pod.DeepCopy()
-	p.Translater.TranslateTo(tPod)
-	tPod.UID = currentPod.UID
-	// this is a bit dangerous since another process could have made changes that the user didn't know about
-	tPod.ResourceVersion = currentPod.ResourceVersion
-
-	// Volumes may refer to resources (configmaps/secrets) from the host cluster
-	// So we need the configuration as calculated during create time
-	tPod.Spec.Volumes = currentPod.Spec.Volumes
-	tPod.Spec.Containers = currentPod.Spec.Containers
-	tPod.Spec.InitContainers = currentPod.Spec.InitContainers
-	tPod.Spec.NodeName = currentPod.Spec.NodeName
-
-	return p.HostClient.Update(ctx, tPod)
+	return p.withRetry(ctx, p.updatePod, pod)
 }
 
-// DeletePod takes a Kubernetes Pod and deletes it from the provider. Once a pod is deleted, the provider is
+func (p *Provider) updatePod(ctx context.Context, pod *v1.Pod) error {
+	p.logger.Debugw("got a request for update pod")
+
+	// Once scheduled a Pod cannot update other fields than the image of the containers, initcontainers and a few others
+	// See: https://kubernetes.io/docs/concepts/workloads/pods/#pod-update-and-replacement
+
+	// Update Pod in the virtual cluster
+
+	var currentVirtualPod v1.Pod
+	if err := p.VirtualClient.Get(ctx, client.ObjectKeyFromObject(pod), &currentVirtualPod); err != nil {
+		return fmt.Errorf("unable to get pod to update from virtual cluster: %w", err)
+	}
+
+	currentVirtualPod.Spec.Containers = updateContainerImages(currentVirtualPod.Spec.Containers, pod.Spec.Containers)
+	currentVirtualPod.Spec.InitContainers = updateContainerImages(currentVirtualPod.Spec.InitContainers, pod.Spec.InitContainers)
+
+	currentVirtualPod.Spec.ActiveDeadlineSeconds = pod.Spec.ActiveDeadlineSeconds
+	currentVirtualPod.Spec.Tolerations = pod.Spec.Tolerations
+
+	// in the virtual cluster we can update also the labels and annotations
+	currentVirtualPod.Annotations = pod.Annotations
+	currentVirtualPod.Labels = pod.Labels
+
+	if err := p.VirtualClient.Update(ctx, &currentVirtualPod); err != nil {
+		return fmt.Errorf("unable to update pod in the virtual cluster: %w", err)
+	}
+
+	// Update Pod in the host cluster
+
+	hostNamespaceName := types.NamespacedName{
+		Namespace: p.ClusterNamespace,
+		Name:      p.Translator.TranslateName(pod.Namespace, pod.Name),
+	}
+
+	var currentHostPod corev1.Pod
+	if err := p.HostClient.Get(ctx, hostNamespaceName, &currentHostPod); err != nil {
+		return fmt.Errorf("unable to get pod to update from host cluster: %w", err)
+	}
+
+	currentHostPod.Spec.Containers = updateContainerImages(currentHostPod.Spec.Containers, pod.Spec.Containers)
+	currentHostPod.Spec.InitContainers = updateContainerImages(currentHostPod.Spec.InitContainers, pod.Spec.InitContainers)
+
+	// update ActiveDeadlineSeconds and Tolerations
+	currentHostPod.Spec.ActiveDeadlineSeconds = pod.Spec.ActiveDeadlineSeconds
+	currentHostPod.Spec.Tolerations = pod.Spec.Tolerations
+
+	if err := p.HostClient.Update(ctx, &currentHostPod); err != nil {
+		return fmt.Errorf("unable to update pod in the host cluster: %w", err)
+	}
+
+	return nil
+}
+
+// updateContainerImages will update the images of the original container images with the same name
+func updateContainerImages(original, updated []v1.Container) []v1.Container {
+	newImages := make(map[string]string)
+
+	for _, c := range updated {
+		newImages[c.Name] = c.Image
+	}
+
+	for i, c := range original {
+		if updatedImage, found := newImages[c.Name]; found {
+			original[i].Image = updatedImage
+		}
+	}
+
+	return original
+}
+
+// DeletePod executes deletePod with retry
+func (p *Provider) DeletePod(ctx context.Context, pod *corev1.Pod) error {
+	return p.withRetry(ctx, p.deletePod, pod)
+}
+
+// deletePod takes a Kubernetes Pod and deletes it from the provider. Once a pod is deleted, the provider is
 // expected to call the NotifyPods callback with a terminal pod status where all the containers are in a terminal
 // state, as well as the pod. DeletePod may be called multiple times for the same pod.
-func (p *Provider) DeletePod(ctx context.Context, pod *corev1.Pod) error {
+func (p *Provider) deletePod(ctx context.Context, pod *corev1.Pod) error {
 	p.logger.Infof("Got request to delete pod %s", pod.Name)
-	hostName := p.Translater.TranslateName(pod.Namespace, pod.Name)
+	hostName := p.Translator.TranslateName(pod.Namespace, pod.Name)
 	err := p.CoreClient.Pods(p.ClusterNamespace).Delete(ctx, hostName, metav1.DeleteOptions{})
 	if err != nil {
 		return fmt.Errorf("unable to delete pod %s/%s: %w", pod.Namespace, pod.Name, err)
 	}
 	if err = p.pruneUnusedVolumes(ctx, pod); err != nil {
-		// note that we don't return an error here. The pod was sucessfully deleted, another process
+		// note that we don't return an error here. The pod was successfully deleted, another process
 		// should clean this without affecting the user
 		p.logger.Errorf("failed to prune leftover volumes for %s/%s: %w, resources may be left", pod.Namespace, pod.Name, err)
 	}
@@ -553,14 +670,14 @@ func (p *Provider) GetPod(ctx context.Context, namespace, name string) (*corev1.
 	p.logger.Debugw("got a request for get pod", "Namespace", namespace, "Name", name)
 	hostNamespaceName := types.NamespacedName{
 		Namespace: p.ClusterNamespace,
-		Name:      p.Translater.TranslateName(namespace, name),
+		Name:      p.Translator.TranslateName(namespace, name),
 	}
 	var pod corev1.Pod
 	err := p.HostClient.Get(ctx, hostNamespaceName, &pod)
 	if err != nil {
 		return nil, fmt.Errorf("error when retrieving pod: %w", err)
 	}
-	p.Translater.TranslateFrom(&pod)
+	p.Translator.TranslateFrom(&pod)
 	return &pod, nil
 }
 
@@ -596,13 +713,22 @@ func (p *Provider) GetPods(ctx context.Context) ([]*corev1.Pod, error) {
 	}
 	retPods := []*corev1.Pod{}
 	for _, pod := range podList.DeepCopy().Items {
-		p.Translater.TranslateFrom(&pod)
+		p.Translator.TranslateFrom(&pod)
 		retPods = append(retPods, &pod)
 	}
 	return retPods, nil
 }
 
-func (p *Provider) configureNetworking(podName, podNamespace string, pod *corev1.Pod) {
+// configureNetworking will inject network information to each pod to connect them to the
+// virtual cluster api server, as well as confiugre DNS information to connect them to the
+// synced coredns on the host cluster.
+func (p *Provider) configureNetworking(podName, podNamespace string, pod *corev1.Pod, serverIP string) {
+	// inject serverIP to hostalias for the pod
+	KubernetesHostAlias := corev1.HostAlias{
+		IP:        serverIP,
+		Hostnames: []string{"kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"},
+	}
+	pod.Spec.HostAliases = append(pod.Spec.HostAliases, KubernetesHostAlias)
 	// inject networking information to the pod's environment variables
 	for i := range pod.Spec.Containers {
 		pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env,
@@ -628,6 +754,31 @@ func (p *Provider) configureNetworking(podName, podNamespace string, pod *corev1
 			},
 		)
 	}
+	// handle init containers as well
+	for i := range pod.Spec.InitContainers {
+		pod.Spec.InitContainers[i].Env = append(pod.Spec.InitContainers[i].Env,
+			corev1.EnvVar{
+				Name:  "KUBERNETES_PORT_443_TCP",
+				Value: "tcp://" + p.serverIP + ":6443",
+			},
+			corev1.EnvVar{
+				Name:  "KUBERNETES_PORT",
+				Value: "tcp://" + p.serverIP + ":6443",
+			},
+			corev1.EnvVar{
+				Name:  "KUBERNETES_PORT_443_TCP_ADDR",
+				Value: p.serverIP,
+			},
+			corev1.EnvVar{
+				Name:  "KUBERNETES_SERVICE_HOST",
+				Value: p.serverIP,
+			},
+			corev1.EnvVar{
+				Name:  "KUBERNETES_SERVICE_PORT",
+				Value: "6443",
+			},
+		)
+	}
 	// injecting cluster DNS IP to the pods except for coredns pod
 	if !strings.HasPrefix(podName, "coredns") {
 		pod.Spec.DNSPolicy = corev1.DNSNone
@@ -640,7 +791,6 @@ func (p *Provider) configureNetworking(podName, podNamespace string, pod *corev1
 			},
 		}
 	}
-
 }
 
 // getSecretsAndConfigmaps retrieves a list of all secrets/configmaps that are in use by a given pod. Useful
@@ -665,3 +815,64 @@ func getSecretsAndConfigmaps(pod *corev1.Pod) ([]string, []string) {
 	}
 	return secrets, configMaps
 }
+
+// configureFieldPathEnv will retrieve all annotations created by the pod mutator webhook
+// to assign env fieldpaths to pods, it will also make sure to change the metadata.name and metadata.namespace to the
+// assigned annotations
+func (p *Provider) configureFieldPathEnv(pod, tPod *v1.Pod) error {
+	// override metadata.name and metadata.namespace with pod annotations
+	for i, container := range pod.Spec.InitContainers {
+		for j, envVar := range container.Env {
+			if envVar.ValueFrom == nil || envVar.ValueFrom.FieldRef == nil {
+				continue
+			}
+
+			fieldPath := envVar.ValueFrom.FieldRef.FieldPath
+
+			if fieldPath == translate.MetadataNameField {
+				envVar.ValueFrom.FieldRef.FieldPath = fmt.Sprintf("metadata.annotations['%s']", translate.ResourceNameAnnotation)
+				pod.Spec.InitContainers[i].Env[j] = envVar
+			}
+			if fieldPath == translate.MetadataNamespaceField {
+				envVar.ValueFrom.FieldRef.FieldPath = fmt.Sprintf("metadata.annotations['%s']", translate.MetadataNamespaceField)
+				pod.Spec.InitContainers[i].Env[j] = envVar
+			}
+		}
+	}
+	for i, container := range pod.Spec.Containers {
+		for j, envVar := range container.Env {
+			if envVar.ValueFrom == nil || envVar.ValueFrom.FieldRef == nil {
+				continue
+			}
+			fieldPath := envVar.ValueFrom.FieldRef.FieldPath
+			if fieldPath == translate.MetadataNameField {
+				envVar.ValueFrom.FieldRef.FieldPath = fmt.Sprintf("metadata.annotations['%s']", translate.ResourceNameAnnotation)
+				pod.Spec.Containers[i].Env[j] = envVar
+			}
+			if fieldPath == translate.MetadataNamespaceField {
+				envVar.ValueFrom.FieldRef.FieldPath = fmt.Sprintf("metadata.annotations['%s']", translate.ResourceNameAnnotation)
+				pod.Spec.Containers[i].Env[j] = envVar
+			}
+		}
+	}
+	for name, value := range pod.Annotations {
+		if strings.Contains(name, webhook.FieldpathField) {
+			containerIndex, envName, err := webhook.ParseFieldPathAnnotationKey(name)
+			if err != nil {
+				return err
+			}
+			// re-adding these envs to the pod
+			tPod.Spec.Containers[containerIndex].Env = append(tPod.Spec.Containers[containerIndex].Env, v1.EnvVar{
+				Name: envName,
+				ValueFrom: &v1.EnvVarSource{
+					FieldRef: &v1.ObjectFieldSelector{
+						FieldPath: value,
+					},
+				},
+			})
+			// removing the annotation from the pod
+			delete(tPod.Annotations, name)
+		}
+	}
+	return nil
+}

k3k-kubelet/provider/token.go

@@ -23,6 +23,12 @@ const (
 func (p *Provider) transformTokens(ctx context.Context, pod, tPod *corev1.Pod) error {
 	p.logger.Infow("transforming token", "Pod", pod.Name, "Namespace", pod.Namespace, "serviceAccountName", pod.Spec.ServiceAccountName)
 
+	// skip this process if the kube-api-access is already removed from the pod
+	// this is needed in case users already adds their own custom tokens like in rancher imported clusters
+	if !isKubeAccessVolumeFound(pod) {
+		return nil
+	}
+
 	virtualSecretName := k3kcontroller.SafeConcatNameWithPrefix(pod.Spec.ServiceAccountName, "token")
 	virtualSecret := virtualSecret(virtualSecretName, pod.Namespace, pod.Spec.ServiceAccountName)
 	if err := p.VirtualClient.Create(ctx, virtualSecret); err != nil {
@@ -46,7 +52,7 @@ func (p *Provider) transformTokens(ctx context.Context, pod, tPod *corev1.Pod) e
 	hostSecret := virtualSecret.DeepCopy()
 	hostSecret.Type = ""
 	hostSecret.Annotations = make(map[string]string)
-	p.Translater.TranslateTo(hostSecret)
+	p.Translator.TranslateTo(hostSecret)
 
 	if err := p.HostClient.Create(ctx, hostSecret); err != nil {
 		if !apierrors.IsAlreadyExists(err) {
@@ -84,12 +90,30 @@ func (p *Provider) translateToken(pod *corev1.Pod, hostSecretName string) {
 	addKubeAccessVolume(pod, hostSecretName)
 }
 
+func isKubeAccessVolumeFound(pod *corev1.Pod) bool {
+	for _, volume := range pod.Spec.Volumes {
+		if strings.HasPrefix(volume.Name, kubeAPIAccessPrefix) {
+			return true
+		}
+	}
+	return false
+}
+
 func removeKubeAccessVolume(pod *corev1.Pod) {
 	for i, volume := range pod.Spec.Volumes {
 		if strings.HasPrefix(volume.Name, kubeAPIAccessPrefix) {
 			pod.Spec.Volumes = append(pod.Spec.Volumes[:i], pod.Spec.Volumes[i+1:]...)
 		}
 	}
+	// init containers
+	for i, container := range pod.Spec.InitContainers {
+		for j, mountPath := range container.VolumeMounts {
+			if strings.HasPrefix(mountPath.Name, kubeAPIAccessPrefix) {
+				pod.Spec.InitContainers[i].VolumeMounts = append(pod.Spec.InitContainers[i].VolumeMounts[:j], pod.Spec.InitContainers[i].VolumeMounts[j+1:]...)
+			}
+		}
+	}
+
 	for i, container := range pod.Spec.Containers {
 		for j, mountPath := range container.VolumeMounts {
 			if strings.HasPrefix(mountPath.Name, kubeAPIAccessPrefix) {
@@ -109,6 +133,14 @@ func addKubeAccessVolume(pod *corev1.Pod, hostSecretName string) {
 			},
 		},
 	})
+
+	for i := range pod.Spec.InitContainers {
+		pod.Spec.InitContainers[i].VolumeMounts = append(pod.Spec.InitContainers[i].VolumeMounts, corev1.VolumeMount{
+			Name:      tokenVolumeName,
+			MountPath: serviceAccountTokenMountPath,
+		})
+	}
+
 	for i := range pod.Spec.Containers {
 		pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, corev1.VolumeMount{
 			Name:      tokenVolumeName,

k3k-kubelet/translate/host.go

@@ -18,14 +18,18 @@ const (
 	// ResourceNamespaceAnnotation is the key for the annotation that contains the original namespace of this
 	// resource in the virtual cluster
 	ResourceNamespaceAnnotation = "k3k.io/namespace"
+	// MetadataNameField is the downwardapi field for object's name
+	MetadataNameField = "metadata.name"
+	// MetadataNamespaceField is the downward field for the object's namespace
+	MetadataNamespaceField = "metadata.namespace"
 )
 
-type ToHostTranslater struct {
+type ToHostTranslator struct {
 	// ClusterName is the name of the virtual cluster whose resources we are
 	// translating to a host cluster
 	ClusterName string
 	// ClusterNamespace is the namespace of the virtual cluster whose resources
-	// we are tranlsating to a host cluster
+	// we are translating to a host cluster
 	ClusterNamespace string
 }
 
@@ -33,7 +37,7 @@ type ToHostTranslater struct {
 // static resources such as configmaps/secrets, and not for things like pods (which can reference other
 // objects). Note that this won't set host-cluster values (like resource version) so when updating you
 // may need to fetch the existing value and do some combination before using this.
-func (t *ToHostTranslater) TranslateTo(obj client.Object) {
+func (t *ToHostTranslator) TranslateTo(obj client.Object) {
 	// owning objects may be in the virtual cluster, but may not be in the host cluster
 	obj.SetOwnerReferences(nil)
 	// add some annotations to make it easier to track source object
@@ -63,7 +67,7 @@ func (t *ToHostTranslater) TranslateTo(obj client.Object) {
 	obj.SetFinalizers(nil)
 }
 
-func (t *ToHostTranslater) TranslateFrom(obj client.Object) {
+func (t *ToHostTranslator) TranslateFrom(obj client.Object) {
 	// owning objects may be in the virtual cluster, but may not be in the host cluster
 	obj.SetOwnerReferences(nil)
 
@@ -91,7 +95,7 @@ func (t *ToHostTranslater) TranslateFrom(obj client.Object) {
 }
 
 // TranslateName returns the name of the resource in the host cluster. Will not update the object with this name.
-func (t *ToHostTranslater) TranslateName(namespace string, name string) string {
+func (t *ToHostTranslator) TranslateName(namespace string, name string) string {
 	// we need to come up with a name which is:
 	// - somewhat connectable to the original resource
 	// - a valid k8s name

main.go

@@ -3,17 +3,20 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 
 	"github.com/go-logr/zapr"
 	"github.com/rancher/k3k/cli/cmds"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/buildinfo"
 	"github.com/rancher/k3k/pkg/controller/cluster"
 	"github.com/rancher/k3k/pkg/controller/clusterset"
 	"github.com/rancher/k3k/pkg/log"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 	"go.uber.org/zap"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
@@ -22,42 +25,43 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 )
 
-const (
-	program   = "k3k"
-	version   = "dev"
-	gitCommit = "HEAD"
-)
-
 var (
-	scheme           = runtime.NewScheme()
-	clusterCIDR      string
-	sharedAgentImage string
-	kubeconfig       string
-	debug            bool
-	logger           *log.Logger
-	flags            = []cli.Flag{
-		cli.StringFlag{
+	scheme                     = runtime.NewScheme()
+	clusterCIDR                string
+	sharedAgentImage           string
+	sharedAgentImagePullPolicy string
+	kubeconfig                 string
+	debug                      bool
+	logger                     *log.Logger
+	flags                      = []cli.Flag{
+		&cli.StringFlag{
 			Name:        "kubeconfig",
-			EnvVar:      "KUBECONFIG",
+			EnvVars:     []string{"KUBECONFIG"},
 			Usage:       "Kubeconfig path",
 			Destination: &kubeconfig,
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "cluster-cidr",
-			EnvVar:      "CLUSTER_CIDR",
+			EnvVars:     []string{"CLUSTER_CIDR"},
 			Usage:       "Cluster CIDR to be added to the networkpolicy of the clustersets",
 			Destination: &clusterCIDR,
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:        "shared-agent-image",
-			EnvVar:      "SHARED_AGENT_IMAGE",
+			EnvVars:     []string{"SHARED_AGENT_IMAGE"},
 			Usage:       "K3K Virtual Kubelet image",
-			Value:       "rancher/k3k:k3k-kubelet-dev",
+			Value:       "rancher/k3k:latest",
 			Destination: &sharedAgentImage,
 		},
-		cli.BoolFlag{
+		&cli.StringFlag{
+			Name:        "shared-agent-pull-policy",
+			EnvVars:     []string{"SHARED_AGENT_PULL_POLICY"},
+			Usage:       "K3K Virtual Kubelet image pull policy must be one of Always, IfNotPresent or Never",
+			Destination: &sharedAgentImagePullPolicy,
+		},
+		&cli.BoolFlag{
 			Name:        "debug",
-			EnvVar:      "DEBUG",
+			EnvVars:     []string{"DEBUG"},
 			Usage:       "Debug level logging",
 			Destination: &debug,
 		},
@@ -73,20 +77,24 @@ func main() {
 	app := cmds.NewApp()
 	app.Flags = flags
 	app.Action = run
-	app.Version = version + " (" + gitCommit + ")"
+	app.Version = buildinfo.Version
 	app.Before = func(clx *cli.Context) error {
+		if err := validate(); err != nil {
+			return err
+		}
 		logger = log.New(debug)
 		return nil
 	}
 	if err := app.Run(os.Args); err != nil {
 		logger.Fatalw("failed to run k3k controller", zap.Error(err))
 	}
-
 }
 
 func run(clx *cli.Context) error {
 	ctx := context.Background()
 
+	logger.Info("Starting k3k - Version: " + buildinfo.Version)
+
 	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
 	if err != nil {
 		return fmt.Errorf("failed to create config from kubeconfig file: %v", err)
@@ -101,24 +109,25 @@ func run(clx *cli.Context) error {
 	}
 
 	ctrlruntimelog.SetLogger(zapr.NewLogger(logger.Desugar().WithOptions(zap.AddCallerSkip(1))))
+
 	logger.Info("adding cluster controller")
-	if err := cluster.Add(ctx, mgr, sharedAgentImage, logger); err != nil {
+	if err := cluster.Add(ctx, mgr, sharedAgentImage, sharedAgentImagePullPolicy); err != nil {
 		return fmt.Errorf("failed to add the new cluster controller: %v", err)
 	}
 
 	logger.Info("adding etcd pod controller")
-	if err := cluster.AddPodController(ctx, mgr, logger); err != nil {
+	if err := cluster.AddPodController(ctx, mgr); err != nil {
 		return fmt.Errorf("failed to add the new cluster controller: %v", err)
 	}
 
 	logger.Info("adding clusterset controller")
-	if err := clusterset.Add(ctx, mgr, clusterCIDR, logger); err != nil {
+	if err := clusterset.Add(ctx, mgr, clusterCIDR); err != nil {
 		return fmt.Errorf("failed to add the clusterset controller: %v", err)
 	}
 
 	if clusterCIDR == "" {
 		logger.Info("adding networkpolicy node controller")
-		if err := clusterset.AddNodeController(ctx, mgr, logger); err != nil {
+		if err := clusterset.AddNodeController(ctx, mgr); err != nil {
 			return fmt.Errorf("failed to add the clusterset node controller: %v", err)
 		}
 	}
@@ -129,3 +138,14 @@ func run(clx *cli.Context) error {
 
 	return nil
 }
+
+func validate() error {
+	if sharedAgentImagePullPolicy != "" {
+		if sharedAgentImagePullPolicy != string(v1.PullAlways) &&
+			sharedAgentImagePullPolicy != string(v1.PullIfNotPresent) &&
+			sharedAgentImagePullPolicy != string(v1.PullNever) {
+			return errors.New("invalid value for shared agent image policy")
+		}
+	}
+	return nil
+}

ops/test

@@ -3,5 +3,7 @@ set -e
 
 cd $(dirname $0)/..
 
-echo Running tests
-go test -cover -tags=test ./...
+if [ -z ${SKIP_TESTS} ]; then
+  echo Running tests
+  go test -cover -tags=test ./...
+fi

pkg/apis/k3k.io/v1alpha1/register.go

@@ -7,11 +7,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
-var SchemeGroupVersion = schema.GroupVersion{Group: k3k.GroupName, Version: "v1alpha1"}
-
 var (
-	SchemBuilder = runtime.NewSchemeBuilder(addKnownTypes)
-	AddToScheme  = SchemBuilder.AddToScheme
+	SchemeGroupVersion = schema.GroupVersion{Group: k3k.GroupName, Version: "v1alpha1"}
+	SchemBuilder       = runtime.NewSchemeBuilder(addKnownTypes)
+	AddToScheme        = SchemBuilder.AddToScheme
 )
 
 func Resource(resource string) schema.GroupResource {

pkg/apis/k3k.io/v1alpha1/set_types.go

@@ -9,7 +9,7 @@ import (
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
-
+// +kubebuilder:object:root=true
 type ClusterSet struct {
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 	metav1.TypeMeta   `json:",inline"`
@@ -77,7 +77,7 @@ type ClusterSetStatus struct {
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
+// +kubebuilder:object:root=true
 type ClusterSetList struct {
 	metav1.ListMeta `json:"metadata,omitempty"`
 	metav1.TypeMeta `json:",inline"`

pkg/apis/k3k.io/v1alpha1/types.go

@@ -7,31 +7,42 @@ import (
 
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:object:root=true
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
-
 type Cluster struct {
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 	metav1.TypeMeta   `json:",inline"`
 
+	// +kubebuilder:default={}
+	// +optional
 	Spec   ClusterSpec   `json:"spec"`
 	Status ClusterStatus `json:"status,omitempty"`
 }
 
 type ClusterSpec struct {
 	// Version is a string representing the Kubernetes version to be used by the virtual nodes.
+	//
+	// +optional
 	Version string `json:"version"`
 
 	// Servers is the number of K3s pods to run in server (controlplane) mode.
+	//
+	// +kubebuilder:default=1
 	// +kubebuilder:validation:XValidation:message="cluster must have at least one server",rule="self >= 1"
+	// +optional
 	Servers *int32 `json:"servers"`
 
 	// Agents is the number of K3s pods to run in agent (worker) mode.
+	//
+	// +kubebuilder:default=0
 	// +kubebuilder:validation:XValidation:message="invalid value for agents",rule="self >= 0"
+	// +optional
 	Agents *int32 `json:"agents"`
 
 	// NodeSelector is the node selector that will be applied to all server/agent pods.
 	// In "shared" mode the node selector will be applied also to the workloads.
+	//
 	// +optional
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
 
@@ -73,14 +84,17 @@ type ClusterSpec struct {
 	Addons []Addon `json:"addons,omitempty"`
 
 	// Mode is the cluster provisioning mode which can be either "shared" or "virtual". Defaults to "shared"
+	//
 	// +kubebuilder:default="shared"
 	// +kubebuilder:validation:Enum=shared;virtual
 	// +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf"
-	Mode ClusterMode `json:"mode"`
+	// +optional
+	Mode ClusterMode `json:"mode,omitempty"`
 
 	// Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data
 	// persistence is guaranteed, so restart of a virtual cluster pod may result in data loss without this field.
-	Persistence *PersistenceConfig `json:"persistence,omitempty"`
+	// +kubebuilder:default={type: "dynamic"}
+	Persistence PersistenceConfig `json:"persistence,omitempty"`
 
 	// Expose contains options for exposing the apiserver inside/outside of the cluster. By default, this is only exposed as a
 	// clusterIP which is relatively secure, but difficult to access outside of the cluster.
@@ -94,9 +108,16 @@ type ClusterSpec struct {
 // ClusterMode is the possible provisioning mode of a Cluster.
 type ClusterMode string
 
+// +kubebuilder:default="dynamic"
+//
+// PersistenceMode is the storage mode of a Cluster.
+type PersistenceMode string
+
 const (
 	SharedClusterMode  = ClusterMode("shared")
 	VirtualClusterMode = ClusterMode("virtual")
+	EphemeralNodeType  = PersistenceMode("ephemeral")
+	DynamicNodesType   = PersistenceMode("dynamic")
 )
 
 type ClusterLimit struct {
@@ -112,7 +133,7 @@ type Addon struct {
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
+// +kubebuilder:object:root=true
 type ClusterList struct {
 	metav1.ListMeta `json:"metadata,omitempty"`
 	metav1.TypeMeta `json:",inline"`
@@ -121,11 +142,10 @@ type ClusterList struct {
 }
 
 type PersistenceConfig struct {
-	// Type can be ephermal, static, dynamic
-	// +kubebuilder:default="ephemeral"
-	Type               string `json:"type"`
-	StorageClassName   string `json:"storageClassName,omitempty"`
-	StorageRequestSize string `json:"storageRequestSize,omitempty"`
+	// +kubebuilder:default="dynamic"
+	Type               PersistenceMode `json:"type"`
+	StorageClassName   *string         `json:"storageClassName,omitempty"`
+	StorageRequestSize string          `json:"storageRequestSize,omitempty"`
 }
 
 type ExposeConfig struct {
@@ -138,8 +158,10 @@ type ExposeConfig struct {
 }
 
 type IngressConfig struct {
-	Enabled          bool   `json:"enabled,omitempty"`
-	IngressClassName string `json:"ingressClassName,omitempty"`
+	// Annotations is a key value map that will enrich the Ingress annotations
+	// +optional
+	Annotations      map[string]string `json:"annotations,omitempty"`
+	IngressClassName string            `json:"ingressClassName,omitempty"`
 }
 
 type LoadBalancerConfig struct {
@@ -147,13 +169,25 @@ type LoadBalancerConfig struct {
 }
 
 type NodePortConfig struct {
-	Enabled bool `json:"enabled"`
+	// ServerPort is the port on each node on which the K3s server service is exposed when type is NodePort.
+	// If not specified, a port will be allocated (default: 30000-32767)
+	// +optional
+	ServerPort *int32 `json:"serverPort,omitempty"`
+	// ServicePort is the port on each node on which the K3s service is exposed when type is NodePort.
+	// If not specified, a port will be allocated (default: 30000-32767)
+	// +optional
+	ServicePort *int32 `json:"servicePort,omitempty"`
+	// ETCDPort is the port on each node on which the ETCD service is exposed when type is NodePort.
+	// If not specified, a port will be allocated (default: 30000-32767)
+	// +optional
+	ETCDPort *int32 `json:"etcdPort,omitempty"`
 }
 
 type ClusterStatus struct {
-	ClusterCIDR string             `json:"clusterCIDR,omitempty"`
-	ServiceCIDR string             `json:"serviceCIDR,omitempty"`
-	ClusterDNS  string             `json:"clusterDNS,omitempty"`
-	TLSSANs     []string           `json:"tlsSANs,omitempty"`
-	Persistence *PersistenceConfig `json:"persistence,omitempty"`
+	HostVersion string            `json:"hostVersion,omitempty"`
+	ClusterCIDR string            `json:"clusterCIDR,omitempty"`
+	ServiceCIDR string            `json:"serviceCIDR,omitempty"`
+	ClusterDNS  string            `json:"clusterDNS,omitempty"`
+	TLSSANs     []string          `json:"tlsSANs,omitempty"`
+	Persistence PersistenceConfig `json:"persistence,omitempty"`
 }

pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go

@@ -201,6 +201,16 @@ func (in *ClusterSetSpec) DeepCopyInto(out *ClusterSetSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.AllowedNodeTypes != nil {
+		in, out := &in.AllowedNodeTypes, &out.AllowedNodeTypes
+		*out = make([]ClusterMode, len(*in))
+		copy(*out, *in)
+	}
+	if in.PodSecurityAdmissionLevel != nil {
+		in, out := &in.PodSecurityAdmissionLevel, &out.PodSecurityAdmissionLevel
+		*out = new(PodSecurityAdmissionLevel)
+		**out = **in
+	}
 	return
 }
 
@@ -262,6 +272,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = new(ClusterLimit)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.TokenSecretRef != nil {
+		in, out := &in.TokenSecretRef, &out.TokenSecretRef
+		*out = new(v1.SecretReference)
+		**out = **in
+	}
 	if in.ServerArgs != nil {
 		in, out := &in.ServerArgs, &out.ServerArgs
 		*out = make([]string, len(*in))
@@ -282,11 +297,7 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = make([]Addon, len(*in))
 		copy(*out, *in)
 	}
-	if in.Persistence != nil {
-		in, out := &in.Persistence, &out.Persistence
-		*out = new(PersistenceConfig)
-		**out = **in
-	}
+	in.Persistence.DeepCopyInto(&out.Persistence)
 	if in.Expose != nil {
 		in, out := &in.Expose, &out.Expose
 		*out = new(ExposeConfig)
@@ -313,11 +324,7 @@ func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
-	if in.Persistence != nil {
-		in, out := &in.Persistence, &out.Persistence
-		*out = new(PersistenceConfig)
-		**out = **in
-	}
+	in.Persistence.DeepCopyInto(&out.Persistence)
 	return
 }
 
@@ -337,7 +344,7 @@ func (in *ExposeConfig) DeepCopyInto(out *ExposeConfig) {
 	if in.Ingress != nil {
 		in, out := &in.Ingress, &out.Ingress
 		*out = new(IngressConfig)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.LoadBalancer != nil {
 		in, out := &in.LoadBalancer, &out.LoadBalancer
@@ -347,7 +354,7 @@ func (in *ExposeConfig) DeepCopyInto(out *ExposeConfig) {
 	if in.NodePort != nil {
 		in, out := &in.NodePort, &out.NodePort
 		*out = new(NodePortConfig)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -365,6 +372,13 @@ func (in *ExposeConfig) DeepCopy() *ExposeConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressConfig) DeepCopyInto(out *IngressConfig) {
 	*out = *in
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	return
 }
 
@@ -397,6 +411,21 @@ func (in *LoadBalancerConfig) DeepCopy() *LoadBalancerConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NodePortConfig) DeepCopyInto(out *NodePortConfig) {
 	*out = *in
+	if in.ServerPort != nil {
+		in, out := &in.ServerPort, &out.ServerPort
+		*out = new(int32)
+		**out = **in
+	}
+	if in.ServicePort != nil {
+		in, out := &in.ServicePort, &out.ServicePort
+		*out = new(int32)
+		**out = **in
+	}
+	if in.ETCDPort != nil {
+		in, out := &in.ETCDPort, &out.ETCDPort
+		*out = new(int32)
+		**out = **in
+	}
 	return
 }
 
@@ -413,6 +442,11 @@ func (in *NodePortConfig) DeepCopy() *NodePortConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PersistenceConfig) DeepCopyInto(out *PersistenceConfig) {
 	*out = *in
+	if in.StorageClassName != nil {
+		in, out := &in.StorageClassName, &out.StorageClassName
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/buildinfo/buildinfo.go

@@ -0,0 +1,3 @@
+package buildinfo
+
+var Version = "dev"

pkg/controller/cluster/agent/agent.go

@@ -1,28 +1,55 @@
 package agent
 
 import (
+	"context"
+	"fmt"
+
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
 const (
 	configName = "agent-config"
 )
 
-type Agent interface {
-	Name() string
-	Config() ctrlruntimeclient.Object
-	Resources() ([]ctrlruntimeclient.Object, error)
+type ResourceEnsurer interface {
+	EnsureResources(context.Context) error
 }
 
-func New(cluster *v1alpha1.Cluster, serviceIP, sharedAgentImage, token string) Agent {
-	if cluster.Spec.Mode == VirtualNodeMode {
-		return NewVirtualAgent(cluster, serviceIP, token)
+type Config struct {
+	cluster *v1alpha1.Cluster
+	client  ctrlruntimeclient.Client
+	scheme  *runtime.Scheme
+}
+
+func NewConfig(cluster *v1alpha1.Cluster, client ctrlruntimeclient.Client, scheme *runtime.Scheme) *Config {
+	return &Config{
+		cluster: cluster,
+		client:  client,
+		scheme:  scheme,
 	}
-	return NewSharedAgent(cluster, serviceIP, sharedAgentImage, token)
 }
 
 func configSecretName(clusterName string) string {
 	return controller.SafeConcatNameWithPrefix(clusterName, configName)
 }
+
+func ensureObject(ctx context.Context, cfg *Config, obj ctrlruntimeclient.Object) error {
+	log := ctrl.LoggerFrom(ctx)
+
+	result, err := controllerutil.CreateOrUpdate(ctx, cfg.client, obj, func() error {
+		return controllerutil.SetControllerReference(cfg.cluster, obj, cfg.scheme)
+	})
+
+	if result != controllerutil.OperationResultNone {
+		key := client.ObjectKeyFromObject(obj)
+		log.Info(fmt.Sprintf("ensuring %T", obj), "key", key, "result, result")
+	}
+
+	return err
+}

pkg/controller/cluster/agent/shared.go

@@ -1,8 +1,10 @@
 package agent
 
 import (
+	"context"
 	"crypto"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"time"
 
@@ -14,8 +16,10 @@ import (
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -26,25 +30,52 @@ const (
 )
 
 type SharedAgent struct {
-	cluster          *v1alpha1.Cluster
-	serviceIP        string
-	sharedAgentImage string
-	token            string
+	*Config
+	serviceIP       string
+	image           string
+	imagePullPolicy string
+	token           string
 }
 
-func NewSharedAgent(cluster *v1alpha1.Cluster, serviceIP, sharedAgentImage, token string) Agent {
+func NewSharedAgent(config *Config, serviceIP, image, imagePullPolicy, token string) *SharedAgent {
 	return &SharedAgent{
-		cluster:          cluster,
-		serviceIP:        serviceIP,
-		sharedAgentImage: sharedAgentImage,
-		token:            token,
+		Config:          config,
+		serviceIP:       serviceIP,
+		image:           image,
+		imagePullPolicy: imagePullPolicy,
+		token:           token,
 	}
 }
 
-func (s *SharedAgent) Config() ctrlruntimeclient.Object {
-	config := sharedAgentData(s.cluster, s.token, s.Name(), s.serviceIP)
+func (s *SharedAgent) Name() string {
+	return controller.SafeConcatNameWithPrefix(s.cluster.Name, SharedNodeAgentName)
+}
 
-	return &v1.Secret{
+func (s *SharedAgent) EnsureResources(ctx context.Context) error {
+	if err := errors.Join(
+		s.config(ctx),
+		s.serviceAccount(ctx),
+		s.role(ctx),
+		s.roleBinding(ctx),
+		s.service(ctx),
+		s.deployment(ctx),
+		s.dnsService(ctx),
+		s.webhookTLS(ctx),
+	); err != nil {
+		return fmt.Errorf("failed to ensure some resources: %w\n", err)
+	}
+
+	return nil
+}
+
+func (s *SharedAgent) ensureObject(ctx context.Context, obj ctrlruntimeclient.Object) error {
+	return ensureObject(ctx, s.Config, obj)
+}
+
+func (s *SharedAgent) config(ctx context.Context) error {
+	config := sharedAgentData(s.cluster, s.Name(), s.token, s.serviceIP)
+
+	configSecret := &v1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Secret",
 			APIVersion: "v1",
@@ -57,44 +88,32 @@ func (s *SharedAgent) Config() ctrlruntimeclient.Object {
 			"config.yaml": []byte(config),
 		},
 	}
+
+	return s.ensureObject(ctx, configSecret)
 }
 
-func sharedAgentData(cluster *v1alpha1.Cluster, token, nodeName, ip string) string {
+func sharedAgentData(cluster *v1alpha1.Cluster, serviceName, token, ip string) string {
+	version := cluster.Spec.Version
+	if cluster.Spec.Version == "" {
+		version = cluster.Status.HostVersion
+	}
 	return fmt.Sprintf(`clusterName: %s
 clusterNamespace: %s
-nodeName: %s
-agentHostname: %s
 serverIP: %s
-token: %s`,
-		cluster.Name, cluster.Namespace, nodeName, nodeName, ip, token)
+serviceName: %s
+token: %s
+version: %s`,
+		cluster.Name, cluster.Namespace, ip, serviceName, token, version)
 }
 
-func (s *SharedAgent) Resources() ([]ctrlruntimeclient.Object, error) {
-	// generate certs for webhook
-	certSecret, err := s.webhookTLS()
-	if err != nil {
-		return nil, err
-	}
-	return []ctrlruntimeclient.Object{
-		s.serviceAccount(),
-		s.role(),
-		s.roleBinding(),
-		s.service(),
-		s.deployment(),
-		s.dnsService(),
-		certSecret}, nil
-}
-
-func (s *SharedAgent) deployment() *apps.Deployment {
-	selector := &metav1.LabelSelector{
-		MatchLabels: map[string]string{
-			"cluster": s.cluster.Name,
-			"type":    "agent",
-			"mode":    "shared",
-		},
+func (s *SharedAgent) deployment(ctx context.Context) error {
+	labels := map[string]string{
+		"cluster": s.cluster.Name,
+		"type":    "agent",
+		"mode":    "shared",
 	}
 
-	return &apps.Deployment{
+	deploy := &apps.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Deployment",
 			APIVersion: "apps/v1",
@@ -102,34 +121,28 @@ func (s *SharedAgent) deployment() *apps.Deployment {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      s.Name(),
 			Namespace: s.cluster.Namespace,
-			Labels:    selector.MatchLabels,
+			Labels:    labels,
 		},
 		Spec: apps.DeploymentSpec{
-			Selector: selector,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
 			Template: v1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: selector.MatchLabels,
+					Labels: labels,
 				},
-				Spec: s.podSpec(selector),
+				Spec: s.podSpec(),
 			},
 		},
 	}
+
+	return s.ensureObject(ctx, deploy)
 }
 
-func (s *SharedAgent) podSpec(affinitySelector *metav1.LabelSelector) v1.PodSpec {
+func (s *SharedAgent) podSpec() v1.PodSpec {
 	var limit v1.ResourceList
 
 	return v1.PodSpec{
-		Affinity: &v1.Affinity{
-			PodAntiAffinity: &v1.PodAntiAffinity{
-				RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
-					{
-						LabelSelector: affinitySelector,
-						TopologyKey:   "kubernetes.io/hostname",
-					},
-				},
-			},
-		},
 		ServiceAccountName: s.Name(),
 		Volumes: []v1.Volume{
 			{
@@ -172,8 +185,8 @@ func (s *SharedAgent) podSpec(affinitySelector *metav1.LabelSelector) v1.PodSpec
 		Containers: []v1.Container{
 			{
 				Name:            s.Name(),
-				Image:           s.sharedAgentImage,
-				ImagePullPolicy: v1.PullAlways,
+				Image:           s.image,
+				ImagePullPolicy: v1.PullPolicy(s.imagePullPolicy),
 				Resources: v1.ResourceRequirements{
 					Limits: limit,
 				},
@@ -181,6 +194,17 @@ func (s *SharedAgent) podSpec(affinitySelector *metav1.LabelSelector) v1.PodSpec
 					"--config",
 					sharedKubeletConfigPath,
 				},
+				Env: []v1.EnvVar{
+					{
+						Name: "AGENT_HOSTNAME",
+						ValueFrom: &v1.EnvVarSource{
+							FieldRef: &v1.ObjectFieldSelector{
+								APIVersion: "v1",
+								FieldPath:  "spec.nodeName",
+							},
+						},
+					},
+				},
 				VolumeMounts: []v1.VolumeMount{
 					{
 						Name:      "config",
@@ -201,11 +225,12 @@ func (s *SharedAgent) podSpec(affinitySelector *metav1.LabelSelector) v1.PodSpec
 					},
 				},
 			},
-		}}
+		},
+	}
 }
 
-func (s *SharedAgent) service() *v1.Service {
-	return &v1.Service{
+func (s *SharedAgent) service(ctx context.Context) error {
+	svc := &v1.Service{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Service",
 			APIVersion: "v1",
@@ -236,16 +261,20 @@ func (s *SharedAgent) service() *v1.Service {
 			},
 		},
 	}
+
+	return s.ensureObject(ctx, svc)
 }
 
-func (s *SharedAgent) dnsService() *v1.Service {
-	return &v1.Service{
+func (s *SharedAgent) dnsService(ctx context.Context) error {
+	dnsServiceName := controller.SafeConcatNameWithPrefix(s.cluster.Name, "kube-dns")
+
+	svc := &v1.Service{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Service",
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      s.DNSName(),
+			Name:      dnsServiceName,
 			Namespace: s.cluster.Namespace,
 		},
 		Spec: v1.ServiceSpec{
@@ -276,10 +305,12 @@ func (s *SharedAgent) dnsService() *v1.Service {
 			},
 		},
 	}
+
+	return s.ensureObject(ctx, svc)
 }
 
-func (s *SharedAgent) serviceAccount() *v1.ServiceAccount {
-	return &v1.ServiceAccount{
+func (s *SharedAgent) serviceAccount(ctx context.Context) error {
+	svcAccount := &v1.ServiceAccount{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "ServiceAccount",
 			APIVersion: "v1",
@@ -289,10 +320,12 @@ func (s *SharedAgent) serviceAccount() *v1.ServiceAccount {
 			Namespace: s.cluster.Namespace,
 		},
 	}
+
+	return s.ensureObject(ctx, svcAccount)
 }
 
-func (s *SharedAgent) role() *rbacv1.Role {
-	return &rbacv1.Role{
+func (s *SharedAgent) role(ctx context.Context) error {
+	role := &rbacv1.Role{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Role",
 			APIVersion: "rbac.authorization.k8s.io/v1",
@@ -314,10 +347,12 @@ func (s *SharedAgent) role() *rbacv1.Role {
 			},
 		},
 	}
+
+	return s.ensureObject(ctx, role)
 }
 
-func (s *SharedAgent) roleBinding() *rbacv1.RoleBinding {
-	return &rbacv1.RoleBinding{
+func (s *SharedAgent) roleBinding(ctx context.Context) error {
+	roleBinding := &rbacv1.RoleBinding{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "RoleBinding",
 			APIVersion: "rbac.authorization.k8s.io/v1",
@@ -339,49 +374,12 @@ func (s *SharedAgent) roleBinding() *rbacv1.RoleBinding {
 			},
 		},
 	}
+
+	return s.ensureObject(ctx, roleBinding)
 }
 
-func (s *SharedAgent) Name() string {
-	return controller.SafeConcatNameWithPrefix(s.cluster.Name, SharedNodeAgentName)
-}
-
-func (s *SharedAgent) DNSName() string {
-	return controller.SafeConcatNameWithPrefix(s.cluster.Name, "kube-dns")
-}
-
-func (s *SharedAgent) webhookTLS() (*v1.Secret, error) {
-	// generate CA CERT/KEY
-	caKeyBytes, err := certutil.MakeEllipticPrivateKeyPEM()
-	if err != nil {
-		return nil, err
-	}
-
-	caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
-	if err != nil {
-		return nil, err
-	}
-
-	cfg := certutil.Config{
-		CommonName: fmt.Sprintf("k3k-webhook-ca@%d", time.Now().Unix()),
-	}
-
-	caCert, err := certutil.NewSelfSignedCACert(cfg, caKey.(crypto.Signer))
-	if err != nil {
-		return nil, err
-	}
-
-	caCertBytes := certutil.EncodeCertPEM(caCert)
-	// generate webhook cert bundle
-	altNames := certs.AddSANs([]string{s.Name(), s.cluster.Name})
-	webhookCert, webhookKey, err := certs.CreateClientCertKey(
-		s.Name(), nil,
-		&altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, time.Hour*24*time.Duration(356),
-		string(caCertBytes),
-		string(caKeyBytes))
-	if err != nil {
-		return nil, err
-	}
-	return &v1.Secret{
+func (s *SharedAgent) webhookTLS(ctx context.Context) error {
+	webhookSecret := &v1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Secret",
 			APIVersion: "v1",
@@ -390,13 +388,80 @@ func (s *SharedAgent) webhookTLS() (*v1.Secret, error) {
 			Name:      WebhookSecretName(s.cluster.Name),
 			Namespace: s.cluster.Namespace,
 		},
-		Data: map[string][]byte{
+	}
+
+	key := client.ObjectKeyFromObject(webhookSecret)
+	if err := s.client.Get(ctx, key, webhookSecret); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return err
+		}
+
+		caPrivateKeyPEM, caCertPEM, err := newWebhookSelfSignedCACerts()
+		if err != nil {
+			return err
+		}
+
+		altNames := []string{s.Name(), s.cluster.Name}
+		webhookCert, webhookKey, err := newWebhookCerts(s.Name(), altNames, caPrivateKeyPEM, caCertPEM)
+		if err != nil {
+			return err
+		}
+
+		webhookSecret.Data = map[string][]byte{
 			"tls.crt": webhookCert,
 			"tls.key": webhookKey,
-			"ca.crt":  caCertBytes,
-			"ca.key":  caKeyBytes,
-		},
-	}, nil
+			"ca.crt":  caCertPEM,
+			"ca.key":  caPrivateKeyPEM,
+		}
+
+		return s.ensureObject(ctx, webhookSecret)
+	}
+
+	// if the webhook secret is found we can skip
+	// we should check for their validity
+	return nil
+}
+
+func newWebhookSelfSignedCACerts() ([]byte, []byte, error) {
+	// generate CA CERT/KEY
+	caPrivateKeyPEM, err := certutil.MakeEllipticPrivateKeyPEM()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caPrivateKey, err := certutil.ParsePrivateKeyPEM(caPrivateKeyPEM)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfg := certutil.Config{
+		CommonName: fmt.Sprintf("k3k-webhook-ca@%d", time.Now().Unix()),
+	}
+
+	caCert, err := certutil.NewSelfSignedCACert(cfg, caPrivateKey.(crypto.Signer))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caCertPEM := certutil.EncodeCertPEM(caCert)
+
+	return caPrivateKeyPEM, caCertPEM, nil
+}
+
+func newWebhookCerts(commonName string, subAltNames []string, caPrivateKey, caCert []byte) ([]byte, []byte, error) {
+	// generate webhook cert bundle
+	altNames := certs.AddSANs(subAltNames)
+	oneYearExpiration := time.Until(time.Now().AddDate(1, 0, 0))
+
+	return certs.CreateClientCertKey(
+		commonName,
+		nil,
+		&altNames,
+		[]x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		oneYearExpiration,
+		string(caCert),
+		string(caPrivateKey),
+	)
 }
 
 func WebhookSecretName(clusterName string) string {

pkg/controller/cluster/agent/shared_test.go

@@ -0,0 +1,114 @@
+package agent
+
+import (
+	"testing"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func Test_sharedAgentData(t *testing.T) {
+	type args struct {
+		cluster     *v1alpha1.Cluster
+		serviceName string
+		ip          string
+		token       string
+	}
+	tests := []struct {
+		name         string
+		args         args
+		expectedData map[string]string
+	}{
+		{
+			name: "simple config",
+			args: args{
+				cluster: &v1alpha1.Cluster{
+					ObjectMeta: v1.ObjectMeta{
+						Name:      "mycluster",
+						Namespace: "ns-1",
+					},
+					Spec: v1alpha1.ClusterSpec{
+						Version: "v1.2.3",
+					},
+				},
+				ip:          "10.0.0.21",
+				serviceName: "service-name",
+				token:       "dnjklsdjnksd892389238",
+			},
+			expectedData: map[string]string{
+				"clusterName":      "mycluster",
+				"clusterNamespace": "ns-1",
+				"serverIP":         "10.0.0.21",
+				"serviceName":      "service-name",
+				"token":            "dnjklsdjnksd892389238",
+				"version":          "v1.2.3",
+			},
+		},
+		{
+			name: "version in status",
+			args: args{
+				cluster: &v1alpha1.Cluster{
+					ObjectMeta: v1.ObjectMeta{
+						Name:      "mycluster",
+						Namespace: "ns-1",
+					},
+					Spec: v1alpha1.ClusterSpec{
+						Version: "v1.2.3",
+					},
+					Status: v1alpha1.ClusterStatus{
+						HostVersion: "v1.3.3",
+					},
+				},
+				ip:          "10.0.0.21",
+				serviceName: "service-name",
+				token:       "dnjklsdjnksd892389238",
+			},
+			expectedData: map[string]string{
+				"clusterName":      "mycluster",
+				"clusterNamespace": "ns-1",
+				"serverIP":         "10.0.0.21",
+				"serviceName":      "service-name",
+				"token":            "dnjklsdjnksd892389238",
+				"version":          "v1.2.3",
+			},
+		},
+		{
+			name: "missing version in spec",
+			args: args{
+				cluster: &v1alpha1.Cluster{
+					ObjectMeta: v1.ObjectMeta{
+						Name:      "mycluster",
+						Namespace: "ns-1",
+					},
+					Status: v1alpha1.ClusterStatus{
+						HostVersion: "v1.3.3",
+					},
+				},
+				ip:          "10.0.0.21",
+				serviceName: "service-name",
+				token:       "dnjklsdjnksd892389238",
+			},
+			expectedData: map[string]string{
+				"clusterName":      "mycluster",
+				"clusterNamespace": "ns-1",
+				"serverIP":         "10.0.0.21",
+				"serviceName":      "service-name",
+				"token":            "dnjklsdjnksd892389238",
+				"version":          "v1.3.3",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config := sharedAgentData(tt.args.cluster, tt.args.serviceName, tt.args.token, tt.args.ip)
+
+			data := make(map[string]string)
+			err := yaml.Unmarshal([]byte(config), data)
+
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expectedData, data)
+		})
+	}
+}

pkg/controller/cluster/agent/virtual.go

@@ -1,9 +1,10 @@
 package agent
 
 import (
+	"context"
+	"errors"
 	"fmt"
 
-	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -18,23 +19,42 @@ const (
 )
 
 type VirtualAgent struct {
-	cluster   *v1alpha1.Cluster
+	*Config
 	serviceIP string
 	token     string
 }
 
-func NewVirtualAgent(cluster *v1alpha1.Cluster, serviceIP, token string) Agent {
+func NewVirtualAgent(config *Config, serviceIP, token string) *VirtualAgent {
 	return &VirtualAgent{
-		cluster:   cluster,
+		Config:    config,
 		serviceIP: serviceIP,
 		token:     token,
 	}
 }
 
-func (v *VirtualAgent) Config() ctrlruntimeclient.Object {
+func (v *VirtualAgent) Name() string {
+	return controller.SafeConcatNameWithPrefix(v.cluster.Name, virtualNodeAgentName)
+}
+
+func (v *VirtualAgent) EnsureResources(ctx context.Context) error {
+	if err := errors.Join(
+		v.config(ctx),
+		v.deployment(ctx),
+	); err != nil {
+		return fmt.Errorf("failed to ensure some resources: %w\n", err)
+	}
+
+	return nil
+}
+
+func (v *VirtualAgent) ensureObject(ctx context.Context, obj ctrlruntimeclient.Object) error {
+	return ensureObject(ctx, v.Config, obj)
+}
+
+func (v *VirtualAgent) config(ctx context.Context) error {
 	config := virtualAgentData(v.serviceIP, v.token)
 
-	return &v1.Secret{
+	configSecret := &v1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Secret",
 			APIVersion: "v1",
@@ -47,10 +67,8 @@ func (v *VirtualAgent) Config() ctrlruntimeclient.Object {
 			"config.yaml": []byte(config),
 		},
 	}
-}
 
-func (v *VirtualAgent) Resources() ([]ctrlruntimeclient.Object, error) {
-	return []ctrlruntimeclient.Object{v.deployment()}, nil
+	return v.ensureObject(ctx, configSecret)
 }
 
 func virtualAgentData(serviceIP, token string) string {
@@ -59,7 +77,7 @@ token: %s
 with-node-id: true`, serviceIP, token)
 }
 
-func (v *VirtualAgent) deployment() *apps.Deployment {
+func (v *VirtualAgent) deployment(ctx context.Context) error {
 	image := controller.K3SImage(v.cluster)
 
 	const name = "k3k-agent"
@@ -70,7 +88,8 @@ func (v *VirtualAgent) deployment() *apps.Deployment {
 			"mode":    "virtual",
 		},
 	}
-	return &apps.Deployment{
+
+	deployment := &apps.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Deployment",
 			APIVersion: "apps/v1",
@@ -91,22 +110,14 @@ func (v *VirtualAgent) deployment() *apps.Deployment {
 			},
 		},
 	}
+
+	return v.ensureObject(ctx, deployment)
 }
 
 func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelector *metav1.LabelSelector) v1.PodSpec {
 	var limit v1.ResourceList
 	args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...)
 	podSpec := v1.PodSpec{
-		Affinity: &v1.Affinity{
-			PodAntiAffinity: &v1.PodAntiAffinity{
-				RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
-					{
-						LabelSelector: affinitySelector,
-						TopologyKey:   "kubernetes.io/hostname",
-					},
-				},
-			},
-		},
 		Volumes: []v1.Volume{
 			{
 				Name: "config",
@@ -161,9 +172,8 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect
 		},
 		Containers: []v1.Container{
 			{
-				Name:            name,
-				Image:           image,
-				ImagePullPolicy: v1.PullAlways,
+				Name:  name,
+				Image: image,
 				SecurityContext: &v1.SecurityContext{
 					Privileged: ptr.To(true),
 				},
@@ -217,7 +227,3 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect
 
 	return podSpec
 }
-
-func (v *VirtualAgent) Name() string {
-	return controller.SafeConcatNameWithPrefix(v.cluster.Name, virtualNodeAgentName)
-}

pkg/controller/cluster/agent/virtual_test.go

@@ -0,0 +1,44 @@
+package agent
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
+)
+
+func Test_virtualAgentData(t *testing.T) {
+	type args struct {
+		serviceIP string
+		token     string
+	}
+	tests := []struct {
+		name         string
+		args         args
+		expectedData map[string]string
+	}{
+		{
+			name: "simple config",
+			args: args{
+				serviceIP: "10.0.0.21",
+				token:     "dnjklsdjnksd892389238",
+			},
+			expectedData: map[string]string{
+				"server":       "https://10.0.0.21:6443",
+				"token":        "dnjklsdjnksd892389238",
+				"with-node-id": "true",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config := virtualAgentData(tt.args.serviceIP, tt.args.token)
+
+			data := make(map[string]string)
+			err := yaml.Unmarshal([]byte(config), data)
+
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expectedData, data)
+		})
+	}
+}

pkg/controller/cluster/cluster.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"strings"
 	"time"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
@@ -12,15 +13,16 @@ import (
 	"github.com/rancher/k3k/pkg/controller/cluster/agent"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
-	"github.com/rancher/k3k/pkg/log"
-	"go.uber.org/zap"
+	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/discovery"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlruntimecontroller "sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -44,85 +46,115 @@ const (
 )
 
 type ClusterReconciler struct {
-	Client           ctrlruntimeclient.Client
-	Scheme           *runtime.Scheme
-	SharedAgentImage string
-	logger           *log.Logger
+	DiscoveryClient            *discovery.DiscoveryClient
+	Client                     ctrlruntimeclient.Client
+	Scheme                     *runtime.Scheme
+	SharedAgentImage           string
+	SharedAgentImagePullPolicy string
 }
 
 // Add adds a new controller to the manager
-func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage string, logger *log.Logger) error {
+func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage, sharedAgentImagePullPolicy string) error {
+	discoveryClient, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig())
+	if err != nil {
+		return err
+	}
+
+	if sharedAgentImage == "" {
+		return errors.New("missing shared agent image")
+	}
+
 	// initialize a new Reconciler
 	reconciler := ClusterReconciler{
-		Client:           mgr.GetClient(),
-		Scheme:           mgr.GetScheme(),
-		SharedAgentImage: sharedAgentImage,
-		logger:           logger.Named(clusterController),
+		DiscoveryClient:            discoveryClient,
+		Client:                     mgr.GetClient(),
+		Scheme:                     mgr.GetScheme(),
+		SharedAgentImage:           sharedAgentImage,
+		SharedAgentImagePullPolicy: sharedAgentImagePullPolicy,
 	}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.Cluster{}).
 		WithOptions(ctrlruntimecontroller.Options{
 			MaxConcurrentReconciles: maxConcurrentReconciles,
 		}).
+		Owns(&apps.StatefulSet{}).
 		Complete(&reconciler)
 }
 
 func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
-	var (
-		cluster v1alpha1.Cluster
-		podList v1.PodList
-	)
-	log := c.logger.With("Cluster", req.NamespacedName)
+	log := ctrl.LoggerFrom(ctx).WithValues("cluster", req.NamespacedName)
+	ctx = ctrl.LoggerInto(ctx, log) // enrich the current logger
+
+	log.Info("reconciling cluster")
+
+	var cluster v1alpha1.Cluster
 	if err := c.Client.Get(ctx, req.NamespacedName, &cluster); err != nil {
-		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
-	}
-	if cluster.DeletionTimestamp.IsZero() {
-		if !controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) {
-			controllerutil.AddFinalizer(&cluster, clusterFinalizerName)
-			if err := c.Client.Update(ctx, &cluster); err != nil {
-				return reconcile.Result{}, err
-			}
-		}
-		log.Info("enqueue cluster")
-		return reconcile.Result{}, c.createCluster(ctx, &cluster, log)
-	}
-
-	// remove finalizer from the server pods and update them.
-	matchingLabels := ctrlruntimeclient.MatchingLabels(map[string]string{"role": "server"})
-	listOpts := &ctrlruntimeclient.ListOptions{Namespace: cluster.Namespace}
-	matchingLabels.ApplyToList(listOpts)
-	if err := c.Client.List(ctx, &podList, listOpts); err != nil {
-		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
-	}
-	for _, pod := range podList.Items {
-		if controllerutil.ContainsFinalizer(&pod, etcdPodFinalizerName) {
-			controllerutil.RemoveFinalizer(&pod, etcdPodFinalizerName)
-			if err := c.Client.Update(ctx, &pod); err != nil {
-				return reconcile.Result{}, err
-			}
-		}
-	}
-
-	if err := c.unbindNodeProxyClusterRole(ctx, &cluster); err != nil {
 		return reconcile.Result{}, err
 	}
 
-	if controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) {
-		// remove finalizer from the cluster and update it.
-		controllerutil.RemoveFinalizer(&cluster, clusterFinalizerName)
+	// if DeletionTimestamp is not Zero -> finalize the object
+	if !cluster.DeletionTimestamp.IsZero() {
+		return c.finalizeCluster(ctx, cluster)
+	}
+
+	// add finalizers
+	if !controllerutil.AddFinalizer(&cluster, clusterFinalizerName) {
 		if err := c.Client.Update(ctx, &cluster); err != nil {
 			return reconcile.Result{}, err
 		}
 	}
-	log.Info("deleting cluster")
+
+	orig := cluster.DeepCopy()
+
+	reconcilerErr := c.reconcileCluster(ctx, &cluster)
+
+	// update Status if needed
+	if !reflect.DeepEqual(orig.Status, cluster.Status) {
+		if err := c.Client.Status().Update(ctx, &cluster); err != nil {
+			return reconcile.Result{}, err
+		}
+	}
+
+	// if there was an error during the reconciliation, return
+	if reconcilerErr != nil {
+		return reconcile.Result{}, reconcilerErr
+	}
+
+	// update Cluster if needed
+	if !reflect.DeepEqual(orig.Spec, cluster.Spec) {
+		if err := c.Client.Update(ctx, &cluster); err != nil {
+			return reconcile.Result{}, err
+		}
+	}
+
 	return reconcile.Result{}, nil
 }
 
-func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1.Cluster, log *zap.SugaredLogger) error {
+func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alpha1.Cluster) error {
+	log := ctrl.LoggerFrom(ctx)
+
+	// if the Version is not specified we will try to use the same Kubernetes version of the host.
+	// This version is stored in the Status object, and it will not be updated if already set.
+	if cluster.Spec.Version == "" && cluster.Status.HostVersion == "" {
+		log.Info("cluster version not set")
+
+		hostVersion, err := c.DiscoveryClient.ServerVersion()
+		if err != nil {
+			return err
+		}
+
+		// update Status HostVersion
+		k8sVersion := strings.Split(hostVersion.GitVersion, "+")[0]
+		cluster.Status.HostVersion = k8sVersion + "-k3s1"
+	}
+
+	// TODO: update status?
 	if err := c.validate(cluster); err != nil {
-		log.Errorw("invalid change", zap.Error(err))
+		log.Error(err, "invalid change")
 		return nil
 	}
+
 	token, err := c.token(ctx, cluster)
 	if err != nil {
 		return err
@@ -130,12 +162,10 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
 
 	s := server.New(cluster, c.Client, token, string(cluster.Spec.Mode))
 
-	if cluster.Spec.Persistence != nil {
-		cluster.Status.Persistence = cluster.Spec.Persistence
-		if cluster.Spec.Persistence.StorageRequestSize == "" {
-			// default to 1G of request size
-			cluster.Status.Persistence.StorageRequestSize = defaultStoragePersistentSize
-		}
+	cluster.Status.Persistence = cluster.Spec.Persistence
+	if cluster.Spec.Persistence.StorageRequestSize == "" {
+		// default to 1G of request size
+		cluster.Status.Persistence.StorageRequestSize = defaultStoragePersistentSize
 	}
 
 	cluster.Status.ClusterCIDR = cluster.Spec.ClusterCIDR
@@ -148,56 +178,63 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
 		cluster.Status.ServiceCIDR = defaultClusterServiceCIDR
 	}
 
-	log.Info("creating cluster service")
-	serviceIP, err := c.createClusterService(ctx, cluster, s)
+	service, err := c.ensureClusterService(ctx, cluster)
 	if err != nil {
 		return err
 	}
+	serviceIP := service.Spec.ClusterIP
 
 	if err := c.createClusterConfigs(ctx, cluster, s, serviceIP); err != nil {
 		return err
 	}
 
-	// creating statefulsets in case the user chose a persistence type other than ephermal
 	if err := c.server(ctx, cluster, s); err != nil {
 		return err
 	}
 
-	if err := c.agent(ctx, cluster, serviceIP, token); err != nil {
+	if err := c.ensureAgent(ctx, cluster, serviceIP, token); err != nil {
 		return err
 	}
 
-	if cluster.Spec.Expose != nil {
-		if cluster.Spec.Expose.Ingress != nil {
-			serverIngress, err := s.Ingress(ctx, c.Client)
-			if err != nil {
-				return err
-			}
-
-			if err := c.Client.Create(ctx, serverIngress); err != nil {
-				if !apierrors.IsAlreadyExists(err) {
-					return err
-				}
-			}
-		}
+	if err := c.ensureIngress(ctx, cluster); err != nil {
+		return err
 	}
 
-	bootstrapSecret, err := bootstrap.Generate(ctx, cluster, serviceIP, token)
+	if err := c.ensureBootstrapSecret(ctx, cluster, serviceIP, token); err != nil {
+		return err
+	}
+
+	return c.bindNodeProxyClusterRole(ctx, cluster)
+}
+
+// ensureBootstrapSecret will create or update the Secret containing the bootstrap data from the k3s server
+func (c *ClusterReconciler) ensureBootstrapSecret(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP, token string) error {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("ensuring bootstrap secret")
+
+	bootstrapData, err := bootstrap.GenerateBootstrapData(ctx, cluster, serviceIP, token)
 	if err != nil {
 		return err
 	}
 
-	if err := c.Client.Create(ctx, bootstrapSecret); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
+	bootstrapSecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
+			Namespace: cluster.Namespace,
+		},
+	}
+
+	_, err = controllerutil.CreateOrUpdate(ctx, c.Client, bootstrapSecret, func() error {
+		if err := controllerutil.SetControllerReference(cluster, bootstrapSecret, c.Scheme); err != nil {
 			return err
 		}
-	}
 
-	if err := c.bindNodeProxyClusterRole(ctx, cluster); err != nil {
-		return err
-	}
-
-	return c.Client.Update(ctx, cluster)
+		bootstrapSecret.Data = map[string][]byte{
+			"bootstrap": bootstrapData,
+		}
+		return nil
+	})
+	return err
 }
 
 func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server, serviceIP string) error {
@@ -234,33 +271,71 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
 	return nil
 }
 
-func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v1alpha1.Cluster, s *server.Server) (string, error) {
-	// create cluster service
-	clusterService := s.Service(cluster)
+func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v1alpha1.Cluster) (*v1.Service, error) {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("ensuring cluster service")
 
-	if err := controllerutil.SetControllerReference(cluster, clusterService, c.Scheme); err != nil {
-		return "", err
-	}
-	if err := c.Client.Create(ctx, clusterService); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
-			return "", err
+	expectedService := server.Service(cluster)
+
+	currentService := expectedService.DeepCopy()
+	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentService, func() error {
+		if err := controllerutil.SetControllerReference(cluster, currentService, c.Scheme); err != nil {
+			return err
 		}
+
+		currentService.Spec = expectedService.Spec
+		return nil
+	})
+	if err != nil {
+		return nil, err
 	}
 
-	var service v1.Service
-
-	objKey := ctrlruntimeclient.ObjectKey{
-		Namespace: cluster.Namespace,
-		Name:      server.ServiceName(cluster.Name),
-	}
-	if err := c.Client.Get(ctx, objKey, &service); err != nil {
-		return "", err
+	key := client.ObjectKeyFromObject(currentService)
+	if result != controllerutil.OperationResultNone {
+		log.Info("cluster service updated", "key", key, "result", result)
 	}
 
-	return service.Spec.ClusterIP, nil
+	return currentService, nil
+}
+
+func (c *ClusterReconciler) ensureIngress(ctx context.Context, cluster *v1alpha1.Cluster) error {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("ensuring cluster ingress")
+
+	expectedServerIngress := server.Ingress(ctx, cluster)
+
+	// delete existing Ingress if Expose or IngressConfig are nil
+	if cluster.Spec.Expose == nil || cluster.Spec.Expose.Ingress == nil {
+		err := c.Client.Delete(ctx, &expectedServerIngress)
+		return client.IgnoreNotFound(err)
+	}
+
+	currentServerIngress := expectedServerIngress.DeepCopy()
+	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerIngress, func() error {
+		if err := controllerutil.SetControllerReference(cluster, currentServerIngress, c.Scheme); err != nil {
+			return err
+		}
+
+		currentServerIngress.Spec = expectedServerIngress.Spec
+		currentServerIngress.Annotations = expectedServerIngress.Annotations
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	key := client.ObjectKeyFromObject(currentServerIngress)
+	if result != controllerutil.OperationResultNone {
+		log.Info("cluster ingress updated", "key", key, "result", result)
+	}
+
+	return nil
 }
 
 func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server) error {
+	log := ctrl.LoggerFrom(ctx)
+
 	// create headless service for the statefulset
 	serverStatefulService := server.StatefulServerService()
 	if err := controllerutil.SetControllerReference(cluster, serverStatefulService, c.Scheme); err != nil {
@@ -271,20 +346,22 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluste
 			return err
 		}
 	}
-	ServerStatefulSet, err := server.StatefulServer(ctx)
+
+	serverStatefulSet, err := server.StatefulServer(ctx)
 	if err != nil {
 		return err
 	}
 
-	if err := controllerutil.SetControllerReference(cluster, ServerStatefulSet, c.Scheme); err != nil {
-		return err
+	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, serverStatefulSet, func() error {
+		return controllerutil.SetControllerReference(cluster, serverStatefulSet, c.Scheme)
+	})
+
+	if result != controllerutil.OperationResultNone {
+		key := client.ObjectKeyFromObject(serverStatefulSet)
+		log.Info("ensuring serverStatefulSet", "key", key, "result", result)
 	}
 
-	if err := c.ensure(ctx, ServerStatefulSet, false); err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }
 
 func (c *ClusterReconciler) bindNodeProxyClusterRole(ctx context.Context, cluster *v1alpha1.Cluster) error {
@@ -313,40 +390,17 @@ func (c *ClusterReconciler) bindNodeProxyClusterRole(ctx context.Context, cluste
 	return c.Client.Update(ctx, clusterRoleBinding)
 }
 
-func (c *ClusterReconciler) unbindNodeProxyClusterRole(ctx context.Context, cluster *v1alpha1.Cluster) error {
-	clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
-	if err := c.Client.Get(ctx, types.NamespacedName{Name: "k3k-node-proxy"}, clusterRoleBinding); err != nil {
-		return fmt.Errorf("failed to get or find k3k-node-proxy ClusterRoleBinding: %w", err)
+func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP, token string) error {
+	config := agent.NewConfig(cluster, c.Client, c.Scheme)
+
+	var agentEnsurer agent.ResourceEnsurer
+	if cluster.Spec.Mode == agent.VirtualNodeMode {
+		agentEnsurer = agent.NewVirtualAgent(config, serviceIP, token)
+	} else {
+		agentEnsurer = agent.NewSharedAgent(config, serviceIP, c.SharedAgentImage, c.SharedAgentImagePullPolicy, token)
 	}
 
-	subjectName := controller.SafeConcatNameWithPrefix(cluster.Name, agent.SharedNodeAgentName)
-
-	var cleanedSubjects []rbacv1.Subject
-	for _, subject := range clusterRoleBinding.Subjects {
-		if subject.Name != subjectName || subject.Namespace != cluster.Namespace {
-			cleanedSubjects = append(cleanedSubjects, subject)
-		}
-	}
-
-	// if no subject was removed, all good
-	if reflect.DeepEqual(clusterRoleBinding.Subjects, cleanedSubjects) {
-		return nil
-	}
-
-	clusterRoleBinding.Subjects = cleanedSubjects
-	return c.Client.Update(ctx, clusterRoleBinding)
-}
-
-func (c *ClusterReconciler) agent(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP, token string) error {
-	agent := agent.New(cluster, serviceIP, c.SharedAgentImage, token)
-	agentsConfig := agent.Config()
-	agentResources, err := agent.Resources()
-	if err != nil {
-		return err
-	}
-	agentResources = append(agentResources, agentsConfig)
-
-	return c.ensureAll(ctx, cluster, agentResources)
+	return agentEnsurer.EnsureResources(ctx)
 }
 
 func (c *ClusterReconciler) validate(cluster *v1alpha1.Cluster) error {
@@ -355,53 +409,3 @@ func (c *ClusterReconciler) validate(cluster *v1alpha1.Cluster) error {
 	}
 	return nil
 }
-
-func (c *ClusterReconciler) ensureAll(ctx context.Context, cluster *v1alpha1.Cluster, objs []ctrlruntimeclient.Object) error {
-	for _, obj := range objs {
-		if err := controllerutil.SetControllerReference(cluster, obj, c.Scheme); err != nil {
-			return err
-		}
-		if err := c.ensure(ctx, obj, false); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (c *ClusterReconciler) ensure(ctx context.Context, obj ctrlruntimeclient.Object, requiresRecreate bool) error {
-	exists := true
-	existingObject := obj.DeepCopyObject().(ctrlruntimeclient.Object)
-	if err := c.Client.Get(ctx, types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}, existingObject); err != nil {
-		if !apierrors.IsNotFound(err) {
-			return fmt.Errorf("failed to get Object(%T): %w", existingObject, err)
-		}
-		exists = false
-	}
-
-	if !exists {
-		// if not exists create object
-		if err := c.Client.Create(ctx, obj); err != nil {
-			return err
-		}
-		return nil
-	}
-	// if exists then apply udpate or recreate if necessary
-	if reflect.DeepEqual(obj.(metav1.Object), existingObject.(metav1.Object)) {
-		return nil
-	}
-
-	if !requiresRecreate {
-		if err := c.Client.Update(ctx, obj); err != nil {
-			return err
-		}
-	} else {
-		// this handles object that needs recreation including configmaps and secrets
-		if err := c.Client.Delete(ctx, obj); err != nil {
-			return err
-		}
-		if err := c.Client.Create(ctx, obj); err != nil {
-			return err
-		}
-	}
-	return nil
-}

pkg/controller/cluster/cluster_finalize.go

@@ -0,0 +1,79 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/cluster/agent"
+	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+func (c *ClusterReconciler) finalizeCluster(ctx context.Context, cluster v1alpha1.Cluster) (reconcile.Result, error) {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("finalizing Cluster")
+
+	// remove finalizer from the server pods and update them.
+	matchingLabels := ctrlruntimeclient.MatchingLabels(map[string]string{"role": "server"})
+	listOpts := &ctrlruntimeclient.ListOptions{Namespace: cluster.Namespace}
+	matchingLabels.ApplyToList(listOpts)
+
+	var podList v1.PodList
+	if err := c.Client.List(ctx, &podList, listOpts); err != nil {
+		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
+	}
+
+	for _, pod := range podList.Items {
+		if controllerutil.ContainsFinalizer(&pod, etcdPodFinalizerName) {
+			controllerutil.RemoveFinalizer(&pod, etcdPodFinalizerName)
+			if err := c.Client.Update(ctx, &pod); err != nil {
+				return reconcile.Result{}, err
+			}
+		}
+	}
+
+	if err := c.unbindNodeProxyClusterRole(ctx, &cluster); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	if controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) {
+		// remove finalizer from the cluster and update it.
+		controllerutil.RemoveFinalizer(&cluster, clusterFinalizerName)
+		if err := c.Client.Update(ctx, &cluster); err != nil {
+			return reconcile.Result{}, err
+		}
+	}
+	return reconcile.Result{}, nil
+}
+
+func (c *ClusterReconciler) unbindNodeProxyClusterRole(ctx context.Context, cluster *v1alpha1.Cluster) error {
+	clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
+	if err := c.Client.Get(ctx, types.NamespacedName{Name: "k3k-node-proxy"}, clusterRoleBinding); err != nil {
+		return fmt.Errorf("failed to get or find k3k-node-proxy ClusterRoleBinding: %w", err)
+	}
+
+	subjectName := controller.SafeConcatNameWithPrefix(cluster.Name, agent.SharedNodeAgentName)
+
+	var cleanedSubjects []rbacv1.Subject
+	for _, subject := range clusterRoleBinding.Subjects {
+		if subject.Name != subjectName || subject.Namespace != cluster.Namespace {
+			cleanedSubjects = append(cleanedSubjects, subject)
+		}
+	}
+
+	// if no subject was removed, all good
+	if reflect.DeepEqual(clusterRoleBinding.Subjects, cleanedSubjects) {
+		return nil
+	}
+
+	clusterRoleBinding.Subjects = cleanedSubjects
+	return c.Client.Update(ctx, clusterRoleBinding)
+}

pkg/controller/cluster/cluster_suite_test.go

@@ -0,0 +1,96 @@
+package cluster_test
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/go-logr/zapr"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller/cluster"
+
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestController(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Cluster Controller Suite")
+}
+
+var (
+	testEnv   *envtest.Environment
+	k8s       *kubernetes.Clientset
+	k8sClient client.Client
+	ctx       context.Context
+	cancel    context.CancelFunc
+)
+
+var _ = BeforeSuite(func() {
+
+	By("bootstrapping test environment")
+	testEnv = &envtest.Environment{
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")},
+		ErrorIfCRDPathMissing: true,
+	}
+	cfg, err := testEnv.Start()
+	Expect(err).NotTo(HaveOccurred())
+
+	k8s, err = kubernetes.NewForConfig(cfg)
+	Expect(err).NotTo(HaveOccurred())
+
+	scheme := buildScheme()
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
+	Expect(err).NotTo(HaveOccurred())
+
+	ctrl.SetLogger(zapr.NewLogger(zap.NewNop()))
+
+	mgr, err := ctrl.NewManager(cfg, ctrl.Options{Scheme: scheme})
+	Expect(err).NotTo(HaveOccurred())
+
+	ctx, cancel = context.WithCancel(context.Background())
+	err = cluster.Add(ctx, mgr, "rancher/k3k-kubelet:latest", "")
+	Expect(err).NotTo(HaveOccurred())
+
+	go func() {
+		defer GinkgoRecover()
+		err = mgr.Start(ctx)
+		Expect(err).NotTo(HaveOccurred(), "failed to run manager")
+	}()
+})
+
+var _ = AfterSuite(func() {
+	cancel()
+
+	By("tearing down the test environment")
+	err := testEnv.Stop()
+	Expect(err).NotTo(HaveOccurred())
+})
+
+func buildScheme() *runtime.Scheme {
+	scheme := runtime.NewScheme()
+
+	err := corev1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+	err = rbacv1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+	err = appsv1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+	err = networkingv1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+	err = v1alpha1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	return scheme
+}

pkg/controller/cluster/cluster_test.go

@@ -0,0 +1,133 @@
+package cluster_test
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller/cluster/server"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Cluster Controller", func() {
+
+	Context("creating a Cluster", func() {
+
+		var (
+			namespace string
+		)
+
+		BeforeEach(func() {
+			createdNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}}
+			err := k8sClient.Create(context.Background(), createdNS)
+			Expect(err).To(Not(HaveOccurred()))
+			namespace = createdNS.Name
+		})
+
+		When("creating a Cluster", func() {
+
+			var cluster *v1alpha1.Cluster
+
+			BeforeEach(func() {
+				cluster = &v1alpha1.Cluster{
+					ObjectMeta: metav1.ObjectMeta{
+						GenerateName: "cluster-",
+						Namespace:    namespace,
+					},
+				}
+
+				err := k8sClient.Create(ctx, cluster)
+				Expect(err).To(Not(HaveOccurred()))
+			})
+
+			It("will be created with some defaults", func() {
+				Expect(cluster.Spec.Mode).To(Equal(v1alpha1.SharedClusterMode))
+				Expect(cluster.Spec.Agents).To(Equal(ptr.To[int32](0)))
+				Expect(cluster.Spec.Servers).To(Equal(ptr.To[int32](1)))
+				Expect(cluster.Spec.Version).To(BeEmpty())
+				// TOFIX
+				//Expect(cluster.Spec.Persistence.Type).To(Equal(v1alpha1.DynamicNodesType))
+
+				serverVersion, err := k8s.DiscoveryClient.ServerVersion()
+				Expect(err).To(Not(HaveOccurred()))
+				expectedHostVersion := fmt.Sprintf("%s-k3s1", serverVersion.GitVersion)
+
+				Eventually(func() string {
+					err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)
+					Expect(err).To(Not(HaveOccurred()))
+					return cluster.Status.HostVersion
+
+				}).
+					WithTimeout(time.Second * 30).
+					WithPolling(time.Second).
+					Should(Equal(expectedHostVersion))
+			})
+
+			When("exposing the cluster with nodePort and custom posrts", func() {
+				It("will have a NodePort service with the specified port exposed", func() {
+					cluster.Spec.Expose = &v1alpha1.ExposeConfig{
+						NodePort: &v1alpha1.NodePortConfig{
+							ServerPort:  ptr.To[int32](30010),
+							ServicePort: ptr.To[int32](30011),
+							ETCDPort:    ptr.To[int32](30012),
+						},
+					}
+
+					err := k8sClient.Update(ctx, cluster)
+					Expect(err).To(Not(HaveOccurred()))
+
+					var service v1.Service
+
+					Eventually(func() v1.ServiceType {
+						serviceKey := client.ObjectKey{
+							Name:      server.ServiceName(cluster.Name),
+							Namespace: cluster.Namespace,
+						}
+
+						err := k8sClient.Get(ctx, serviceKey, &service)
+						Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
+						return service.Spec.Type
+					}).
+						WithTimeout(time.Second * 30).
+						WithPolling(time.Second).
+						Should(Equal(v1.ServiceTypeNodePort))
+
+					servicePorts := service.Spec.Ports
+					Expect(servicePorts).NotTo(BeEmpty())
+					Expect(servicePorts).To(HaveLen(3))
+
+					Expect(servicePorts).To(ContainElement(
+						And(
+							HaveField("Name", "k3s-server-port"),
+							HaveField("Port", BeEquivalentTo(6443)),
+							HaveField("NodePort", BeEquivalentTo(30010)),
+						),
+					))
+					Expect(servicePorts).To(ContainElement(
+						And(
+							HaveField("Name", "k3s-service-port"),
+							HaveField("Port", BeEquivalentTo(443)),
+							HaveField("NodePort", BeEquivalentTo(30011)),
+						),
+					))
+					Expect(servicePorts).To(ContainElement(
+						And(
+							HaveField("Name", "k3s-etcd-port"),
+							HaveField("Port", BeEquivalentTo(2379)),
+							HaveField("NodePort", BeEquivalentTo(30012)),
+						),
+					))
+				})
+			})
+		})
+	})
+})

pkg/controller/cluster/pod.go

@@ -15,10 +15,8 @@ import (
 	"github.com/rancher/k3k/pkg/controller/certs"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
-	"github.com/rancher/k3k/pkg/log"
 	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
 	clientv3 "go.etcd.io/etcd/client/v3"
-	"go.uber.org/zap"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -41,16 +39,14 @@ const (
 type PodReconciler struct {
 	Client ctrlruntimeclient.Client
 	Scheme *runtime.Scheme
-	logger *log.Logger
 }
 
 // Add adds a new controller to the manager
-func AddPodController(ctx context.Context, mgr manager.Manager, logger *log.Logger) error {
+func AddPodController(ctx context.Context, mgr manager.Manager) error {
 	// initialize a new Reconciler
 	reconciler := PodReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
-		logger: logger.Named(podController),
 	}
 
 	return ctrl.NewControllerManagedBy(mgr).
@@ -63,7 +59,8 @@ func AddPodController(ctx context.Context, mgr manager.Manager, logger *log.Logg
 }
 
 func (p *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
-	log := p.logger.With("Pod", req.NamespacedName)
+	log := ctrl.LoggerFrom(ctx).WithValues("statefulset", req.NamespacedName)
+	ctx = ctrl.LoggerInto(ctx, log) // enrich the current logger
 
 	s := strings.Split(req.Name, "-")
 	if len(s) < 1 {
@@ -74,7 +71,7 @@ func (p *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r
 	}
 	clusterName := s[1]
 	var cluster v1alpha1.Cluster
-	if err := p.Client.Get(ctx, types.NamespacedName{Name: clusterName}, &cluster); err != nil {
+	if err := p.Client.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: req.Namespace}, &cluster); err != nil {
 		if !apierrors.IsNotFound(err) {
 			return reconcile.Result{}, err
 		}
@@ -87,23 +84,31 @@ func (p *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r
 	if err := p.Client.List(ctx, &podList, listOpts); err != nil {
 		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
 	}
+	if len(podList.Items) == 1 {
+		return reconcile.Result{}, nil
+	}
 	for _, pod := range podList.Items {
-		log.Info("Handle etcd server pod")
-		if err := p.handleServerPod(ctx, cluster, &pod, log); err != nil {
+		if err := p.handleServerPod(ctx, cluster, &pod); err != nil {
 			return reconcile.Result{}, err
 		}
 	}
 	return reconcile.Result{}, nil
 }
 
-func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cluster, pod *v1.Pod, log *zap.SugaredLogger) error {
-	if _, ok := pod.Labels["role"]; ok {
-		if pod.Labels["role"] != "server" {
-			return nil
-		}
-	} else {
+func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cluster, pod *v1.Pod) error {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("handling server pod")
+
+	role, found := pod.Labels["role"]
+	if !found {
 		return fmt.Errorf("server pod has no role label")
 	}
+
+	if role != "server" {
+		log.V(1).Info("pod has a different role: " + role)
+		return nil
+	}
+
 	// if etcd pod is marked for deletion then we need to remove it from the etcd member list before deletion
 	if !pod.DeletionTimestamp.IsZero() {
 		// check if cluster is deleted then remove the finalizer from the pod
@@ -116,7 +121,7 @@ func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cl
 			}
 			return nil
 		}
-		tlsConfig, err := p.getETCDTLS(ctx, &cluster, log)
+		tlsConfig, err := p.getETCDTLS(ctx, &cluster)
 		if err != nil {
 			return err
 		}
@@ -131,9 +136,10 @@ func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cl
 			return err
 		}
 
-		if err := removePeer(ctx, client, pod.Name, pod.Status.PodIP, log); err != nil {
+		if err := removePeer(ctx, client, pod.Name, pod.Status.PodIP); err != nil {
 			return err
 		}
+
 		// remove our finalizer from the list and update it.
 		if controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) {
 			controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName)
@@ -150,13 +156,16 @@ func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cl
 	return nil
 }
 
-func (p *PodReconciler) getETCDTLS(ctx context.Context, cluster *v1alpha1.Cluster, log *zap.SugaredLogger) (*tls.Config, error) {
-	log.Infow("generating etcd TLS client certificate", "Cluster", cluster.Name, "Namespace", cluster.Namespace)
+func (p *PodReconciler) getETCDTLS(ctx context.Context, cluster *v1alpha1.Cluster) (*tls.Config, error) {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("generating etcd TLS client certificate", "cluster", cluster)
+
 	token, err := p.clusterToken(ctx, cluster)
 	if err != nil {
 		return nil, err
 	}
 	endpoint := server.ServiceName(cluster.Name) + "." + cluster.Namespace
+
 	var b *bootstrap.ControlRuntimeBootstrap
 	if err := retry.OnError(k3kcontroller.Backoff, func(err error) bool {
 		return true
@@ -191,7 +200,10 @@ func (p *PodReconciler) getETCDTLS(ctx context.Context, cluster *v1alpha1.Cluste
 }
 
 // removePeer removes a peer from the cluster. The peer name and IP address must both match.
-func removePeer(ctx context.Context, client *clientv3.Client, name, address string, log *zap.SugaredLogger) error {
+func removePeer(ctx context.Context, client *clientv3.Client, name, address string) error {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("removing peer from cluster", "name", name, "address", address)
+
 	ctx, cancel := context.WithTimeout(ctx, memberRemovalTimeout)
 	defer cancel()
 	members, err := client.MemberList(ctx)
@@ -208,8 +220,9 @@ func removePeer(ctx context.Context, client *clientv3.Client, name, address stri
 			if err != nil {
 				return err
 			}
+
 			if u.Hostname() == address {
-				log.Infow("Removing member from etcd", "name", member.Name, "id", member.ID, "address", address)
+				log.Info("removing member from etcd", "name", member.Name, "id", member.ID, "address", address)
 				_, err := client.MemberRemove(ctx, member.ID)
 				if errors.Is(err, rpctypes.ErrGRPCMemberNotFound) {
 					return nil

pkg/controller/cluster/server/bootstrap/bootstrap.go

@@ -5,14 +5,16 @@ import (
 	"crypto/tls"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller"
 	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/util/retry"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type ControlRuntimeBootstrap struct {
@@ -32,47 +34,17 @@ type content struct {
 // Generate generates the bootstrap for the cluster:
 // 1- use the server token to get the bootstrap data from k3s
 // 2- save the bootstrap data as a secret
-func Generate(ctx context.Context, cluster *v1alpha1.Cluster, ip, token string) (*v1.Secret, error) {
-	var bootstrap *ControlRuntimeBootstrap
-	if err := retry.OnError(controller.Backoff, func(err error) bool {
-		return true
-	}, func() error {
-		var err error
-		bootstrap, err = requestBootstrap(token, ip)
-		return err
-	}); err != nil {
-		return nil, err
+func GenerateBootstrapData(ctx context.Context, cluster *v1alpha1.Cluster, ip, token string) ([]byte, error) {
+	bootstrap, err := requestBootstrap(token, ip)
+	if err != nil {
+		return nil, fmt.Errorf("failed to request bootstrap secret: %w", err)
 	}
 
 	if err := decodeBootstrap(bootstrap); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to decode bootstrap secret: %w", err)
 	}
 
-	bootstrapData, err := json.Marshal(bootstrap)
-	if err != nil {
-		return nil, err
-	}
-	return &v1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Secret",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
-			Namespace: cluster.Namespace,
-			OwnerReferences: []metav1.OwnerReference{
-				{
-					APIVersion: cluster.APIVersion,
-					Kind:       cluster.Kind,
-					Name:       cluster.Name,
-					UID:        cluster.UID,
-				},
-			},
-		},
-		Data: map[string][]byte{
-			"bootstrap": bootstrapData,
-		},
-	}, nil
+	return json.Marshal(bootstrap)
 
 }
 
@@ -171,3 +143,24 @@ func DecodedBootstrap(token, ip string) (*ControlRuntimeBootstrap, error) {
 
 	return bootstrap, nil
 }
+
+func GetFromSecret(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster) (*ControlRuntimeBootstrap, error) {
+	key := types.NamespacedName{
+		Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
+		Namespace: cluster.Namespace,
+	}
+
+	var bootstrapSecret v1.Secret
+	if err := client.Get(ctx, key, &bootstrapSecret); err != nil {
+		return nil, err
+	}
+
+	bootstrapData := bootstrapSecret.Data["bootstrap"]
+	if bootstrapData == nil {
+		return nil, errors.New("empty bootstrap")
+	}
+
+	var bootstrap ControlRuntimeBootstrap
+	err := json.Unmarshal(bootstrapData, &bootstrap)
+	return &bootstrap, err
+}

pkg/controller/cluster/server/ingress.go

@@ -3,90 +3,84 @@ package server
 import (
 	"context"
 
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
+	"k8s.io/utils/ptr"
 )
 
 const (
-	wildcardDNS = ".sslip.io"
-
-	nginxSSLPassthroughAnnotation  = "nginx.ingress.kubernetes.io/ssl-passthrough"
-	nginxBackendProtocolAnnotation = "nginx.ingress.kubernetes.io/backend-protocol"
-	nginxSSLRedirectAnnotation     = "nginx.ingress.kubernetes.io/ssl-redirect"
-
-	serverPort = 6443
-	etcdPort   = 2379
+	servicePort = 443
+	serverPort  = 6443
+	etcdPort    = 2379
 )
 
-func (s *Server) Ingress(ctx context.Context, client client.Client) (*networkingv1.Ingress, error) {
-	addresses, err := controller.Addresses(ctx, client)
-	if err != nil {
-		return nil, err
-	}
-	ingressRules := s.ingressRules(addresses)
-	ingress := &networkingv1.Ingress{
+func IngressName(clusterName string) string {
+	return controller.SafeConcatNameWithPrefix(clusterName, "ingress")
+}
+
+func Ingress(ctx context.Context, cluster *v1alpha1.Cluster) networkingv1.Ingress {
+	ingress := networkingv1.Ingress{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Ingress",
 			APIVersion: "networking.k8s.io/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      controller.SafeConcatNameWithPrefix(s.cluster.Name, "ingress"),
-			Namespace: s.cluster.Namespace,
+			Name:      IngressName(cluster.Name),
+			Namespace: cluster.Namespace,
 		},
 		Spec: networkingv1.IngressSpec{
-			IngressClassName: &s.cluster.Spec.Expose.Ingress.IngressClassName,
-			Rules:            ingressRules,
+			Rules: ingressRules(cluster),
 		},
 	}
 
-	configureIngressOptions(ingress, s.cluster.Spec.Expose.Ingress.IngressClassName)
+	if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
+		ingressConfig := cluster.Spec.Expose.Ingress
 
-	return ingress, nil
+		if ingressConfig.IngressClassName != "" {
+			ingress.Spec.IngressClassName = ptr.To(ingressConfig.IngressClassName)
+		}
+
+		if ingressConfig.Annotations != nil {
+			ingress.Annotations = ingressConfig.Annotations
+		}
+	}
+
+	return ingress
 }
 
-func (s *Server) ingressRules(addresses []string) []networkingv1.IngressRule {
+func ingressRules(cluster *v1alpha1.Cluster) []networkingv1.IngressRule {
 	var ingressRules []networkingv1.IngressRule
-	pathTypePrefix := networkingv1.PathTypePrefix
-	for _, address := range addresses {
-		rule := networkingv1.IngressRule{
-			Host: s.cluster.Name + "." + address + wildcardDNS,
-			IngressRuleValue: networkingv1.IngressRuleValue{
-				HTTP: &networkingv1.HTTPIngressRuleValue{
-					Paths: []networkingv1.HTTPIngressPath{
-						{
-							Path:     "/",
-							PathType: &pathTypePrefix,
-							Backend: networkingv1.IngressBackend{
-								Service: &networkingv1.IngressServiceBackend{
-									Name: ServiceName(s.cluster.Name),
-									Port: networkingv1.ServiceBackendPort{
-										Number: serverPort,
-									},
-								},
-							},
-						},
-					},
+
+	if cluster.Spec.Expose == nil || cluster.Spec.Expose.Ingress == nil {
+		return ingressRules
+	}
+
+	path := networkingv1.HTTPIngressPath{
+		Path:     "/",
+		PathType: ptr.To(networkingv1.PathTypePrefix),
+		Backend: networkingv1.IngressBackend{
+			Service: &networkingv1.IngressServiceBackend{
+				Name: ServiceName(cluster.Name),
+				Port: networkingv1.ServiceBackendPort{
+					Number: serverPort,
 				},
 			},
-		}
-		ingressRules = append(ingressRules, rule)
+		},
+	}
+
+	hosts := cluster.Spec.TLSSANs
+	for _, host := range hosts {
+		ingressRules = append(ingressRules, networkingv1.IngressRule{
+			Host: host,
+			IngressRuleValue: networkingv1.IngressRuleValue{
+				HTTP: &networkingv1.HTTPIngressRuleValue{
+					Paths: []networkingv1.HTTPIngressPath{path},
+				},
+			},
+		})
 	}
 
 	return ingressRules
 }
-
-// configureIngressOptions will configure the ingress object by
-// adding tls passthrough capabilities and TLS needed annotations
-// it depends on the ingressclassname to configure each ingress
-// TODO: add treafik support through ingresstcproutes
-func configureIngressOptions(ingress *networkingv1.Ingress, ingressClassName string) {
-	// initial support for nginx ingress via annotations
-	if ingressClassName == "nginx" {
-		ingress.Annotations = make(map[string]string)
-		ingress.Annotations[nginxSSLPassthroughAnnotation] = "true"
-		ingress.Annotations[nginxSSLRedirectAnnotation] = "true"
-		ingress.Annotations[nginxBackendProtocolAnnotation] = "HTTPS"
-	}
-}

pkg/controller/cluster/server/server.go

@@ -1,8 +1,10 @@
 package server
 
 import (
+	"bytes"
 	"context"
 	"strings"
+	"text/template"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller"
@@ -23,9 +25,7 @@ const (
 	configName         = "server-config"
 	initConfigName     = "init-server-config"
 
-	ServerPort        = 6443
-	EphermalNodesType = "ephermal"
-	DynamicNodesType  = "dynamic"
+	ServerPort = 6443
 )
 
 // Server
@@ -45,7 +45,7 @@ func New(cluster *v1alpha1.Cluster, client client.Client, token, mode string) *S
 	}
 }
 
-func (s *Server) podSpec(image, name string, persistent bool, affinitySelector *metav1.LabelSelector) v1.PodSpec {
+func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) v1.PodSpec {
 	var limit v1.ResourceList
 	if s.cluster.Spec.Limit != nil && s.cluster.Spec.Limit.ServerLimit != nil {
 		limit = s.cluster.Spec.Limit.ServerLimit
@@ -53,16 +53,6 @@ func (s *Server) podSpec(image, name string, persistent bool, affinitySelector *
 	podSpec := v1.PodSpec{
 		NodeSelector:      s.cluster.Spec.NodeSelector,
 		PriorityClassName: s.cluster.Spec.PriorityClass,
-		Affinity: &v1.Affinity{
-			PodAntiAffinity: &v1.PodAntiAffinity{
-				RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
-					{
-						LabelSelector: affinitySelector,
-						TopologyKey:   "kubernetes.io/hostname",
-					},
-				},
-			},
-		},
 		Volumes: []v1.Volume{
 			{
 				Name: "initconfig",
@@ -116,6 +106,12 @@ func (s *Server) podSpec(image, name string, persistent bool, affinitySelector *
 					EmptyDir: &v1.EmptyDirVolumeSource{},
 				},
 			},
+			{
+				Name: "varlibkubelet",
+				VolumeSource: v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{},
+				},
+			},
 		},
 		Containers: []v1.Container{
 			{
@@ -133,17 +129,14 @@ func (s *Server) podSpec(image, name string, persistent bool, affinitySelector *
 							},
 						},
 					},
-				},
-				Command: []string{
-					"/bin/sh",
-					"-c",
-					`
-					if [ ${POD_NAME: -1} == 0 ]; then 
-						/bin/k3s server --config /opt/rancher/k3s/init/config.yaml ` + strings.Join(s.cluster.Spec.ServerArgs, " ") + `
-					else 
-						/bin/k3s server --config /opt/rancher/k3s/server/config.yaml ` + strings.Join(s.cluster.Spec.ServerArgs, " ") + `
-					fi   
-					`,
+					{
+						Name: "POD_IP",
+						ValueFrom: &v1.EnvVarSource{
+							FieldRef: &v1.ObjectFieldSelector{
+								FieldPath: "status.podIP",
+							},
+						},
+					},
 				},
 				VolumeMounts: []v1.VolumeMount{
 					{
@@ -191,15 +184,15 @@ func (s *Server) podSpec(image, name string, persistent bool, affinitySelector *
 		},
 	}
 
+	cmd := []string{
+		"/bin/sh",
+		"-c",
+		startupCmd,
+	}
+
+	podSpec.Containers[0].Command = cmd
 	if !persistent {
 		podSpec.Volumes = append(podSpec.Volumes, v1.Volume{
-
-			Name: "varlibkubelet",
-			VolumeSource: v1.VolumeSource{
-				EmptyDir: &v1.EmptyDirVolumeSource{},
-			},
-		}, v1.Volume{
-
 			Name: "varlibrancherk3s",
 			VolumeSource: v1.VolumeSource{
 				EmptyDir: &v1.EmptyDirVolumeSource{},
@@ -231,7 +224,8 @@ func (s *Server) podSpec(image, name string, persistent bool, affinitySelector *
 func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error) {
 	var (
 		replicas   int32
-		pvClaims   []v1.PersistentVolumeClaim
+		pvClaim    v1.PersistentVolumeClaim
+		err        error
 		persistent bool
 	)
 	image := controller.K3SImage(s.cluster)
@@ -239,48 +233,9 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
 
 	replicas = *s.cluster.Spec.Servers
 
-	if s.cluster.Spec.Persistence != nil && s.cluster.Spec.Persistence.Type != EphermalNodesType {
+	if s.cluster.Spec.Persistence.Type == v1alpha1.DynamicNodesType {
 		persistent = true
-		pvClaims = []v1.PersistentVolumeClaim{
-			{
-				TypeMeta: metav1.TypeMeta{
-					Kind:       "PersistentVolumeClaim",
-					APIVersion: "v1",
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "varlibrancherk3s",
-					Namespace: s.cluster.Namespace,
-				},
-				Spec: v1.PersistentVolumeClaimSpec{
-					AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
-					StorageClassName: &s.cluster.Spec.Persistence.StorageClassName,
-					Resources: v1.VolumeResourceRequirements{
-						Requests: v1.ResourceList{
-							"storage": resource.MustParse(s.cluster.Spec.Persistence.StorageRequestSize),
-						},
-					},
-				},
-			},
-			{
-				TypeMeta: metav1.TypeMeta{
-					Kind:       "PersistentVolumeClaim",
-					APIVersion: "v1",
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "varlibkubelet",
-					Namespace: s.cluster.Namespace,
-				},
-				Spec: v1.PersistentVolumeClaimSpec{
-					Resources: v1.VolumeResourceRequirements{
-						Requests: v1.ResourceList{
-							"storage": resource.MustParse(s.cluster.Spec.Persistence.StorageRequestSize),
-						},
-					},
-					AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
-					StorageClassName: &s.cluster.Spec.Persistence.StorageClassName,
-				},
-			},
-		}
+		pvClaim = s.setupDynamicPersistence()
 	}
 
 	var volumes []v1.Volume
@@ -347,11 +302,15 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
 		},
 	}
 
-	podSpec := s.podSpec(image, name, persistent, &selector)
+	startupCommand, err := s.setupStartCommand()
+	if err != nil {
+		return nil, err
+	}
+	podSpec := s.podSpec(image, name, persistent, startupCommand)
 	podSpec.Volumes = append(podSpec.Volumes, volumes...)
 	podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...)
 
-	return &apps.StatefulSet{
+	ss := &apps.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "StatefulSet",
 			APIVersion: "apps/v1",
@@ -362,10 +321,9 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
 			Labels:    selector.MatchLabels,
 		},
 		Spec: apps.StatefulSetSpec{
-			Replicas:             &replicas,
-			ServiceName:          headlessServiceName(s.cluster.Name),
-			Selector:             &selector,
-			VolumeClaimTemplates: pvClaims,
+			Replicas:    &replicas,
+			ServiceName: headlessServiceName(s.cluster.Name),
+			Selector:    &selector,
 			Template: v1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: selector.MatchLabels,
@@ -373,5 +331,55 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
 				Spec: podSpec,
 			},
 		},
-	}, nil
+	}
+	if s.cluster.Spec.Persistence.Type == v1alpha1.DynamicNodesType {
+		ss.Spec.VolumeClaimTemplates = []v1.PersistentVolumeClaim{pvClaim}
+	}
+
+	return ss, nil
+}
+
+func (s *Server) setupDynamicPersistence() v1.PersistentVolumeClaim {
+	return v1.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "PersistentVolumeClaim",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "varlibrancherk3s",
+			Namespace: s.cluster.Namespace,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
+			StorageClassName: s.cluster.Spec.Persistence.StorageClassName,
+			Resources: v1.VolumeResourceRequirements{
+				Requests: v1.ResourceList{
+					"storage": resource.MustParse(s.cluster.Status.Persistence.StorageRequestSize),
+				},
+			},
+		},
+	}
+
+}
+
+func (s *Server) setupStartCommand() (string, error) {
+	var output bytes.Buffer
+
+	tmpl := singleServerTemplate
+	if *s.cluster.Spec.Servers > 1 {
+		tmpl = HAServerTemplate
+	}
+	tmplCmd, err := template.New("").Parse(tmpl)
+	if err != nil {
+		return "", err
+	}
+	if err := tmplCmd.Execute(&output, map[string]string{
+		"ETCD_DIR":      "/var/lib/rancher/k3s/server/db/etcd",
+		"INIT_CONFIG":   "/opt/rancher/k3s/init/config.yaml",
+		"SERVER_CONFIG": "/opt/rancher/k3s/server/config.yaml",
+		"EXTRA_ARGS":    strings.Join(s.cluster.Spec.ServerArgs, " "),
+	}); err != nil {
+		return "", err
+	}
+	return output.String(), nil
 }

pkg/controller/cluster/server/service.go

@@ -5,47 +5,72 @@ import (
 	"github.com/rancher/k3k/pkg/controller"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
-func (s *Server) Service(cluster *v1alpha1.Cluster) *v1.Service {
-	serviceType := v1.ServiceTypeClusterIP
-	if cluster.Spec.Expose != nil {
-		if cluster.Spec.Expose.NodePort != nil {
-			if cluster.Spec.Expose.NodePort.Enabled {
-				serviceType = v1.ServiceTypeNodePort
-			}
-		}
-	}
-
-	return &v1.Service{
+func Service(cluster *v1alpha1.Cluster) *v1.Service {
+	service := &v1.Service{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Service",
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      ServiceName(s.cluster.Name),
+			Name:      ServiceName(cluster.Name),
 			Namespace: cluster.Namespace,
 		},
 		Spec: v1.ServiceSpec{
-			Type: serviceType,
+			Type: v1.ServiceTypeClusterIP,
 			Selector: map[string]string{
 				"cluster": cluster.Name,
 				"role":    "server",
 			},
-			Ports: []v1.ServicePort{
-				{
-					Name:     "k3s-server-port",
-					Protocol: v1.ProtocolTCP,
-					Port:     serverPort,
-				},
-				{
-					Name:     "k3s-etcd-port",
-					Protocol: v1.ProtocolTCP,
-					Port:     etcdPort,
-				},
-			},
 		},
 	}
+
+	k3sServerPort := v1.ServicePort{
+		Name:     "k3s-server-port",
+		Protocol: v1.ProtocolTCP,
+		Port:     serverPort,
+	}
+
+	k3sServicePort := v1.ServicePort{
+		Name:       "k3s-service-port",
+		Protocol:   v1.ProtocolTCP,
+		Port:       servicePort,
+		TargetPort: intstr.FromInt(serverPort),
+	}
+
+	etcdPort := v1.ServicePort{
+		Name:     "k3s-etcd-port",
+		Protocol: v1.ProtocolTCP,
+		Port:     etcdPort,
+	}
+
+	if cluster.Spec.Expose != nil {
+		nodePortConfig := cluster.Spec.Expose.NodePort
+		if nodePortConfig != nil {
+			service.Spec.Type = v1.ServiceTypeNodePort
+
+			if nodePortConfig.ServerPort != nil {
+				k3sServerPort.NodePort = *nodePortConfig.ServerPort
+			}
+			if nodePortConfig.ServicePort != nil {
+				k3sServicePort.NodePort = *nodePortConfig.ServicePort
+			}
+			if nodePortConfig.ETCDPort != nil {
+				etcdPort.NodePort = *nodePortConfig.ETCDPort
+			}
+		}
+	}
+
+	service.Spec.Ports = append(
+		service.Spec.Ports,
+		k3sServicePort,
+		etcdPort,
+		k3sServerPort,
+	)
+
+	return service
 }
 
 func (s *Server) StatefulServerService() *v1.Service {
@@ -71,6 +96,12 @@ func (s *Server) StatefulServerService() *v1.Service {
 					Protocol: v1.ProtocolTCP,
 					Port:     serverPort,
 				},
+				{
+					Name:       "k3s-service-port",
+					Protocol:   v1.ProtocolTCP,
+					Port:       servicePort,
+					TargetPort: intstr.FromInt(serverPort),
+				},
 				{
 					Name:     "k3s-etcd-port",
 					Protocol: v1.ProtocolTCP,

pkg/controller/cluster/server/template.go

@@ -0,0 +1,16 @@
+package server
+
+var singleServerTemplate string = `
+if [ -d "{{.ETCD_DIR}}" ]; then
+	# if directory exists then it means its not an initial run
+	/bin/k3s server --cluster-reset --config  {{.INIT_CONFIG}} {{.EXTRA_ARGS}}
+fi
+rm -f /var/lib/rancher/k3s/server/db/reset-flag
+/bin/k3s server --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}}`
+
+var HAServerTemplate string = ` 
+if [ ${POD_NAME: -1} == 0 ] && [ ! -d "{{.ETCD_DIR}}" ]; then
+	/bin/k3s server --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}}
+else 
+	/bin/k3s server --config {{.SERVER_CONFIG}} {{.EXTRA_ARGS}}
+fi`

pkg/controller/cluster/token.go

@@ -12,6 +12,8 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
@@ -35,15 +37,16 @@ func (c *ClusterReconciler) token(ctx context.Context, cluster *v1alpha1.Cluster
 }
 
 func (c *ClusterReconciler) ensureTokenSecret(ctx context.Context, cluster *v1alpha1.Cluster) (string, error) {
+	log := ctrl.LoggerFrom(ctx)
+
 	// check if the secret is already created
-	var (
-		tokenSecret v1.Secret
-		nn          = types.NamespacedName{
-			Name:      TokenSecretName(cluster.Name),
-			Namespace: cluster.Namespace,
-		}
-	)
-	if err := c.Client.Get(ctx, nn, &tokenSecret); err != nil {
+	key := types.NamespacedName{
+		Name:      TokenSecretName(cluster.Name),
+		Namespace: cluster.Namespace,
+	}
+
+	var tokenSecret v1.Secret
+	if err := c.Client.Get(ctx, key, &tokenSecret); err != nil {
 		if !apierrors.IsNotFound(err) {
 			return "", err
 		}
@@ -52,19 +55,26 @@ func (c *ClusterReconciler) ensureTokenSecret(ctx context.Context, cluster *v1al
 	if tokenSecret.Data != nil {
 		return string(tokenSecret.Data["token"]), nil
 	}
-	c.logger.Info("Token secret is not specified, creating a random token")
+
+	log.Info("Token secret is not specified, creating a random token")
+
 	token, err := random(16)
 	if err != nil {
 		return "", err
 	}
+
 	tokenSecret = TokenSecretObj(token, cluster.Name, cluster.Namespace)
-	if err := controllerutil.SetControllerReference(cluster, &tokenSecret, c.Scheme); err != nil {
-		return "", err
+	key = client.ObjectKeyFromObject(&tokenSecret)
+
+	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, &tokenSecret, func() error {
+		return controllerutil.SetControllerReference(cluster, &tokenSecret, c.Scheme)
+	})
+
+	if result != controllerutil.OperationResultNone {
+		log.Info("ensuring tokenSecret", "key", key, "result", result)
 	}
-	if err := c.ensure(ctx, &tokenSecret, false); err != nil {
-		return "", err
-	}
-	return token, nil
+
+	return token, err
 
 }
 

pkg/controller/clusterset/clusterset.go

@@ -7,8 +7,6 @@ import (
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	k3kcontroller "github.com/rancher/k3k/pkg/controller"
-	"github.com/rancher/k3k/pkg/log"
-	"go.uber.org/zap"
 	v1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -37,17 +35,15 @@ type ClusterSetReconciler struct {
 	Client      ctrlruntimeclient.Client
 	Scheme      *runtime.Scheme
 	ClusterCIDR string
-	logger      *log.Logger
 }
 
 // Add adds a new controller to the manager
-func Add(ctx context.Context, mgr manager.Manager, clusterCIDR string, logger *log.Logger) error {
+func Add(ctx context.Context, mgr manager.Manager, clusterCIDR string) error {
 	// initialize a new Reconciler
 	reconciler := ClusterSetReconciler{
 		Client:      mgr.GetClient(),
 		Scheme:      mgr.GetScheme(),
 		ClusterCIDR: clusterCIDR,
-		logger:      logger.Named(clusterSetController),
 	}
 
 	return ctrl.NewControllerManagedBy(mgr).
@@ -121,22 +117,23 @@ func namespaceLabelsPredicate() predicate.Predicate {
 }
 
 func (c *ClusterSetReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
-	log := c.logger.With("ClusterSet", req.NamespacedName)
+	log := ctrl.LoggerFrom(ctx).WithValues("clusterset", req.NamespacedName)
+	ctx = ctrl.LoggerInto(ctx, log) // enrich the current logger
 
 	var clusterSet v1alpha1.ClusterSet
 	if err := c.Client.Get(ctx, req.NamespacedName, &clusterSet); err != nil {
 		return reconcile.Result{}, client.IgnoreNotFound(err)
 	}
 
-	if err := c.reconcileNetworkPolicy(ctx, log, &clusterSet); err != nil {
+	if err := c.reconcileNetworkPolicy(ctx, &clusterSet); err != nil {
 		return reconcile.Result{}, err
 	}
 
-	if err := c.reconcileNamespacePodSecurityLabels(ctx, log, &clusterSet); err != nil {
+	if err := c.reconcileNamespacePodSecurityLabels(ctx, &clusterSet); err != nil {
 		return reconcile.Result{}, err
 	}
 
-	if err := c.reconcileClusters(ctx, log, &clusterSet); err != nil {
+	if err := c.reconcileClusters(ctx, &clusterSet); err != nil {
 		return reconcile.Result{}, err
 	}
 
@@ -164,7 +161,8 @@ func (c *ClusterSetReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	return reconcile.Result{}, nil
 }
 
-func (c *ClusterSetReconciler) reconcileNetworkPolicy(ctx context.Context, log *zap.SugaredLogger, clusterSet *v1alpha1.ClusterSet) error {
+func (c *ClusterSetReconciler) reconcileNetworkPolicy(ctx context.Context, clusterSet *v1alpha1.ClusterSet) error {
+	log := ctrl.LoggerFrom(ctx)
 	log.Info("reconciling NetworkPolicy")
 
 	networkPolicy, err := netpol(ctx, c.ClusterCIDR, clusterSet, c.Client)
@@ -258,7 +256,8 @@ func netpol(ctx context.Context, clusterCIDR string, clusterSet *v1alpha1.Cluste
 	}, nil
 }
 
-func (c *ClusterSetReconciler) reconcileNamespacePodSecurityLabels(ctx context.Context, log *zap.SugaredLogger, clusterSet *v1alpha1.ClusterSet) error {
+func (c *ClusterSetReconciler) reconcileNamespacePodSecurityLabels(ctx context.Context, clusterSet *v1alpha1.ClusterSet) error {
+	log := ctrl.LoggerFrom(ctx)
 	log.Info("reconciling Namespace")
 
 	var ns v1.Namespace
@@ -293,7 +292,7 @@ func (c *ClusterSetReconciler) reconcileNamespacePodSecurityLabels(ctx context.C
 	}
 
 	if !reflect.DeepEqual(ns.Labels, newLabels) {
-		log.Debug("labels changed, updating namespace")
+		log.V(1).Info("labels changed, updating namespace")
 
 		ns.Labels = newLabels
 		return c.Client.Update(ctx, &ns)
@@ -301,7 +300,8 @@ func (c *ClusterSetReconciler) reconcileNamespacePodSecurityLabels(ctx context.C
 	return nil
 }
 
-func (c *ClusterSetReconciler) reconcileClusters(ctx context.Context, log *zap.SugaredLogger, clusterSet *v1alpha1.ClusterSet) error {
+func (c *ClusterSetReconciler) reconcileClusters(ctx context.Context, clusterSet *v1alpha1.ClusterSet) error {
+	log := ctrl.LoggerFrom(ctx)
 	log.Info("reconciling Clusters")
 
 	var clusters v1alpha1.ClusterList

pkg/controller/clusterset/clusterset_suite_test.go

@@ -5,9 +5,9 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/go-logr/zapr"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller/clusterset"
-	"github.com/rancher/k3k/pkg/log"
 
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
@@ -51,10 +51,10 @@ var _ = BeforeSuite(func() {
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{Scheme: scheme})
 	Expect(err).NotTo(HaveOccurred())
 
-	ctx, cancel = context.WithCancel(context.Background())
-	nopLogger := &log.Logger{SugaredLogger: zap.NewNop().Sugar()}
+	ctrl.SetLogger(zapr.NewLogger(zap.NewNop()))
 
-	err = clusterset.Add(ctx, mgr, "", nopLogger)
+	ctx, cancel = context.WithCancel(context.Background())
+	err = clusterset.Add(ctx, mgr, "")
 	Expect(err).NotTo(HaveOccurred())
 
 	go func() {

pkg/controller/clusterset/clusterset_test.go

@@ -239,7 +239,7 @@ var _ = Describe("ClusterSet Controller", func() {
 
 		})
 
-		When("created specifing the mode", func() {
+		When("created specifying the mode", func() {
 			It("should have the 'virtual' mode if specified", func() {
 				clusterSet := &v1alpha1.ClusterSet{
 					ObjectMeta: v1.ObjectMeta{
@@ -306,7 +306,7 @@ var _ = Describe("ClusterSet Controller", func() {
 			})
 		})
 
-		When("created specifing the podSecurityAdmissionLevel", func() {
+		When("created specifying the podSecurityAdmissionLevel", func() {
 			It("should add and update the proper pod-security labels to the namespace", func() {
 				var (
 					privileged = v1alpha1.PrivilegedPodSecurityAdmissionLevel
@@ -488,8 +488,8 @@ var _ = Describe("ClusterSet Controller", func() {
 					},
 					Spec: v1alpha1.ClusterSpec{
 						Mode:    v1alpha1.SharedClusterMode,
-						Servers: ptr.To(int32(1)),
-						Agents:  ptr.To(int32(0)),
+						Servers: ptr.To[int32](1),
+						Agents:  ptr.To[int32](0),
 					},
 				}
 
@@ -529,8 +529,8 @@ var _ = Describe("ClusterSet Controller", func() {
 					},
 					Spec: v1alpha1.ClusterSpec{
 						Mode:    v1alpha1.SharedClusterMode,
-						Servers: ptr.To(int32(1)),
-						Agents:  ptr.To(int32(0)),
+						Servers: ptr.To[int32](1),
+						Agents:  ptr.To[int32](0),
 					},
 				}
 
@@ -570,8 +570,8 @@ var _ = Describe("ClusterSet Controller", func() {
 					},
 					Spec: v1alpha1.ClusterSpec{
 						Mode:         v1alpha1.SharedClusterMode,
-						Servers:      ptr.To(int32(1)),
-						Agents:       ptr.To(int32(0)),
+						Servers:      ptr.To[int32](1),
+						Agents:       ptr.To[int32](0),
 						NodeSelector: map[string]string{"label-1": "value-1"},
 					},
 				}
@@ -645,8 +645,8 @@ var _ = Describe("ClusterSet Controller", func() {
 					},
 					Spec: v1alpha1.ClusterSpec{
 						Mode:    v1alpha1.SharedClusterMode,
-						Servers: ptr.To(int32(1)),
-						Agents:  ptr.To(int32(0)),
+						Servers: ptr.To[int32](1),
+						Agents:  ptr.To[int32](0),
 					},
 				}
 

pkg/controller/clusterset/node.go

@@ -4,8 +4,6 @@ import (
 	"context"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	"github.com/rancher/k3k/pkg/log"
-	"go.uber.org/zap"
 	v1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -24,16 +22,14 @@ type NodeReconciler struct {
 	Client      ctrlruntimeclient.Client
 	Scheme      *runtime.Scheme
 	ClusterCIDR string
-	logger      *log.Logger
 }
 
 // AddNodeController adds a new controller to the manager
-func AddNodeController(ctx context.Context, mgr manager.Manager, logger *log.Logger) error {
+func AddNodeController(ctx context.Context, mgr manager.Manager) error {
 	// initialize a new Reconciler
 	reconciler := NodeReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
-		logger: logger.Named(nodeController),
 	}
 
 	return ctrl.NewControllerManagedBy(mgr).
@@ -46,7 +42,11 @@ func AddNodeController(ctx context.Context, mgr manager.Manager, logger *log.Log
 }
 
 func (n *NodeReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
-	log := n.logger.With("Node", req.NamespacedName)
+	log := ctrl.LoggerFrom(ctx).WithValues("node", req.NamespacedName)
+	ctx = ctrl.LoggerInto(ctx, log) // enrich the current logger
+
+	log.Info("reconciling node")
+
 	var clusterSetList v1alpha1.ClusterSetList
 	if err := n.Client.List(ctx, &clusterSetList); err != nil {
 		return reconcile.Result{}, err
@@ -56,26 +56,33 @@ func (n *NodeReconciler) Reconcile(ctx context.Context, req reconcile.Request) (
 		return reconcile.Result{}, nil
 	}
 
-	if err := n.ensureNetworkPolicies(ctx, clusterSetList, log); err != nil {
+	if err := n.ensureNetworkPolicies(ctx, clusterSetList); err != nil {
 		return reconcile.Result{}, err
 	}
 
 	return reconcile.Result{}, nil
 }
 
-func (n *NodeReconciler) ensureNetworkPolicies(ctx context.Context, clusterSetList v1alpha1.ClusterSetList, log *zap.SugaredLogger) error {
+func (n *NodeReconciler) ensureNetworkPolicies(ctx context.Context, clusterSetList v1alpha1.ClusterSetList) error {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("ensuring network policies")
+
 	var setNetworkPolicy *networkingv1.NetworkPolicy
 	for _, cs := range clusterSetList.Items {
 		if cs.Spec.DisableNetworkPolicy {
 			continue
 		}
+
+		log = log.WithValues("clusterset", cs.Namespace+"/"+cs.Name)
+		log.Info("updating NetworkPolicy for ClusterSet")
+
 		var err error
-		log.Infow("Updating NetworkPolicy for ClusterSet", "name", cs.Name, "namespace", cs.Namespace)
 		setNetworkPolicy, err = netpol(ctx, "", &cs, n.Client)
 		if err != nil {
 			return err
 		}
-		log.Debugw("New NetworkPolicy for clusterset", "name", cs.Name, "namespace", cs.Namespace)
+
+		log.Info("new NetworkPolicy for clusterset")
 		if err := n.Client.Update(ctx, setNetworkPolicy); err != nil {
 			return err
 		}

pkg/controller/controller.go

@@ -1,16 +1,13 @@
 package controller
 
 import (
-	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"strings"
 	"time"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
-	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -27,42 +24,19 @@ var Backoff = wait.Backoff{
 	Jitter:   0.1,
 }
 
+// K3SImage returns the rancher/k3s image tagged with the specified Version.
+// If Version is empty it will use with the same k8s version of the host cluster,
+// stored in the Status object. It will return the untagged version as last fallback.
 func K3SImage(cluster *v1alpha1.Cluster) string {
-	return k3SImageName + ":" + cluster.Spec.Version
-}
-func nodeAddress(node *v1.Node) string {
-	var externalIP string
-	var internalIP string
-
-	for _, ip := range node.Status.Addresses {
-		if ip.Type == "ExternalIP" && ip.Address != "" {
-			externalIP = ip.Address
-			break
-		}
-		if ip.Type == "InternalIP" && ip.Address != "" {
-			internalIP = ip.Address
-		}
-	}
-	if externalIP != "" {
-		return externalIP
+	if cluster.Spec.Version != "" {
+		return k3SImageName + ":" + cluster.Spec.Version
 	}
 
-	return internalIP
-}
-
-// return all the nodes external addresses, if not found then return internal addresses
-func Addresses(ctx context.Context, client ctrlruntimeclient.Client) ([]string, error) {
-	var nodeList v1.NodeList
-	if err := client.List(ctx, &nodeList); err != nil {
-		return nil, err
+	if cluster.Status.HostVersion != "" {
+		return k3SImageName + ":" + cluster.Status.HostVersion
 	}
 
-	addresses := make([]string, len(nodeList.Items))
-	for _, node := range nodeList.Items {
-		addresses = append(addresses, nodeAddress(&node))
-	}
-
-	return addresses, nil
+	return k3SImageName
 }
 
 // SafeConcatNameWithPrefix runs the SafeConcatName with extra prefix.

pkg/controller/kubeconfig/kubeconfig.go

@@ -3,8 +3,6 @@ package kubeconfig
 import (
 	"context"
 	"crypto/x509"
-	"encoding/json"
-	"errors"
 	"fmt"
 	"time"
 
@@ -15,8 +13,9 @@ import (
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
 	v1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/apiserver/pkg/authentication/user"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -28,60 +27,45 @@ type KubeConfig struct {
 	ExpiryDate time.Duration
 }
 
-func (k *KubeConfig) Extract(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, hostServerIP string) ([]byte, error) {
-	nn := types.NamespacedName{
-		Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
-		Namespace: cluster.Namespace,
+func New() *KubeConfig {
+	return &KubeConfig{
+		CN:         controller.AdminCommonName,
+		ORG:        []string{user.SystemPrivilegedGroup},
+		ExpiryDate: 0,
 	}
-
-	var bootstrapSecret v1.Secret
-	if err := client.Get(ctx, nn, &bootstrapSecret); err != nil {
-		return nil, err
-	}
-
-	bootstrapData := bootstrapSecret.Data["bootstrap"]
-	if bootstrapData == nil {
-		return nil, errors.New("empty bootstrap")
-	}
-
-	var bootstrap bootstrap.ControlRuntimeBootstrap
-	if err := json.Unmarshal(bootstrapData, &bootstrap); err != nil {
-		return nil, err
-	}
-
-	adminCert, adminKey, err := certs.CreateClientCertKey(
-		k.CN, k.ORG,
-		&k.AltNames, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, k.ExpiryDate,
-		bootstrap.ClientCA.Content,
-		bootstrap.ClientCAKey.Content)
-	if err != nil {
-		return nil, err
-	}
-	// get the server service to extract the right IP
-	nn = types.NamespacedName{
-		Name:      server.ServiceName(cluster.Name),
-		Namespace: cluster.Namespace,
-	}
-
-	var k3kService v1.Service
-	if err := client.Get(ctx, nn, &k3kService); err != nil {
-		return nil, err
-	}
-
-	url := fmt.Sprintf("https://%s:%d", k3kService.Spec.ClusterIP, server.ServerPort)
-	if k3kService.Spec.Type == v1.ServiceTypeNodePort {
-		nodePort := k3kService.Spec.Ports[0].NodePort
-		url = fmt.Sprintf("https://%s:%d", hostServerIP, nodePort)
-	}
-	kubeconfigData, err := kubeconfig(url, []byte(bootstrap.ServerCA.Content), adminCert, adminKey)
-	if err != nil {
-		return nil, err
-	}
-
-	return kubeconfigData, nil
 }
 
-func kubeconfig(url string, serverCA, clientCert, clientKey []byte) ([]byte, error) {
+func (k *KubeConfig) Extract(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, hostServerIP string) (*clientcmdapi.Config, error) {
+	bootstrapData, err := bootstrap.GetFromSecret(ctx, client, cluster)
+	if err != nil {
+		return nil, err
+	}
+	serverCACert := []byte(bootstrapData.ServerCA.Content)
+
+	adminCert, adminKey, err := certs.CreateClientCertKey(
+		k.CN,
+		k.ORG,
+		&k.AltNames,
+		[]x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		k.ExpiryDate,
+		bootstrapData.ClientCA.Content,
+		bootstrapData.ClientCAKey.Content,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	url, err := getURLFromService(ctx, client, cluster, hostServerIP)
+	if err != nil {
+		return nil, err
+	}
+
+	config := NewConfig(url, serverCACert, adminCert, adminKey)
+
+	return config, nil
+}
+
+func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi.Config {
 	config := clientcmdapi.NewConfig()
 
 	cluster := clientcmdapi.NewCluster()
@@ -101,10 +85,41 @@ func kubeconfig(url string, serverCA, clientCert, clientKey []byte) ([]byte, err
 	config.Contexts["default"] = context
 	config.CurrentContext = "default"
 
-	kubeconfig, err := clientcmd.Write(*config)
-	if err != nil {
-		return nil, err
+	return config
+}
+
+func getURLFromService(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, hostServerIP string) (string, error) {
+	// get the server service to extract the right IP
+	key := types.NamespacedName{
+		Name:      server.ServiceName(cluster.Name),
+		Namespace: cluster.Namespace,
 	}
 
-	return kubeconfig, nil
+	var k3kService v1.Service
+	if err := client.Get(ctx, key, &k3kService); err != nil {
+		return "", err
+	}
+
+	url := fmt.Sprintf("https://%s:%d", k3kService.Spec.ClusterIP, server.ServerPort)
+
+	if k3kService.Spec.Type == v1.ServiceTypeNodePort {
+		nodePort := k3kService.Spec.Ports[0].NodePort
+		url = fmt.Sprintf("https://%s:%d", hostServerIP, nodePort)
+	}
+
+	expose := cluster.Spec.Expose
+	if expose != nil && expose.Ingress != nil {
+		var k3kIngress networkingv1.Ingress
+		ingressKey := types.NamespacedName{
+			Name:      server.IngressName(cluster.Name),
+			Namespace: cluster.Namespace,
+		}
+
+		if err := client.Get(ctx, ingressKey, &k3kIngress); err != nil {
+			return "", err
+		}
+		url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host)
+	}
+
+	return url, nil
 }

scripts/build

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e pipefail
+
+TAG=$(git describe --tag --always --match="v[0-9]*")
+
+if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
+    TAG="${TAG}-dirty"
+fi
+
+LDFLAGS="-X \"github.com/rancher/k3k/pkg/buildinfo.Version=${TAG}\""
+
+echo "Building k3k..."
+echo "Current TAG: ${TAG}"
+
+export CGO_ENABLED=0
+GOOS=linux GOARCH=amd64 go build -ldflags="${LDFLAGS}" -o bin/k3k
+GOOS=linux GOARCH=amd64 go build -ldflags="${LDFLAGS}" -o bin/k3k-kubelet ./k3k-kubelet
+
+# build the cli for the local OS and ARCH
+go build -ldflags="${LDFLAGS}" -o bin/k3kcli ./cli
+
+docker build -f package/Dockerfile -t rancher/k3k:dev -t rancher/k3k:${TAG} .
+docker build -f package/Dockerfile.kubelet -t rancher/k3k-kubelet:dev -t rancher/k3k-kubelet:${TAG} .

tests/cluster_test.go

@@ -0,0 +1,344 @@
+package k3k_test
+
+import (
+	"context"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/rancher/k3k/k3k-kubelet/translate"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = When("k3k is installed", func() {
+	It("is in Running status", func() {
+
+		// check that the controller is running
+		Eventually(func() bool {
+			opts := v1.ListOptions{LabelSelector: "app.kubernetes.io/name=k3k"}
+			podList, err := k8s.CoreV1().Pods("k3k-system").List(context.Background(), opts)
+
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(podList.Items).To(Not(BeEmpty()))
+
+			var isRunning bool
+			for _, pod := range podList.Items {
+				if pod.Status.Phase == corev1.PodRunning {
+					isRunning = true
+					break
+				}
+			}
+
+			return isRunning
+		}).
+			WithTimeout(time.Second * 10).
+			WithPolling(time.Second).
+			Should(BeTrue())
+	})
+})
+
+var _ = When("a ephemeral cluster is installed", func() {
+
+	var namespace string
+
+	BeforeEach(func() {
+		createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
+		createdNS, err := k8s.CoreV1().Namespaces().Create(context.Background(), createdNS, v1.CreateOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+		namespace = createdNS.Name
+	})
+
+	It("can create a nginx pod", func() {
+		ctx := context.Background()
+
+		cluster := v1alpha1.Cluster{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "mycluster",
+				Namespace: namespace,
+			},
+			Spec: v1alpha1.ClusterSpec{
+				TLSSANs: []string{hostIP},
+				Expose: &v1alpha1.ExposeConfig{
+					NodePort: &v1alpha1.NodePortConfig{},
+				},
+				Persistence: v1alpha1.PersistenceConfig{
+					Type: v1alpha1.EphemeralNodeType,
+				},
+			},
+		}
+
+		By(fmt.Sprintf("Creating virtual cluster %s/%s", cluster.Namespace, cluster.Name))
+		NewVirtualCluster(cluster)
+
+		By("Waiting to get a kubernetes client for the virtual cluster")
+		virtualK8sClient := NewVirtualK8sClient(cluster)
+
+		nginxPod := &corev1.Pod{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "nginx",
+				Namespace: "default",
+			},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{{
+					Name:  "nginx",
+					Image: "nginx",
+				}},
+			},
+		}
+		nginxPod, err := virtualK8sClient.CoreV1().Pods(nginxPod.Namespace).Create(ctx, nginxPod, v1.CreateOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+
+		// check that the nginx Pod is up and running in the host cluster
+		Eventually(func() bool {
+			//labelSelector := fmt.Sprintf("%s=%s", translate.ClusterNameLabel, cluster.Namespace)
+			podList, err := k8s.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{})
+			Expect(err).To(Not(HaveOccurred()))
+
+			for _, pod := range podList.Items {
+				resourceName := pod.Annotations[translate.ResourceNameAnnotation]
+				resourceNamespace := pod.Annotations[translate.ResourceNamespaceAnnotation]
+
+				if resourceName == nginxPod.Name && resourceNamespace == nginxPod.Namespace {
+					fmt.Fprintf(GinkgoWriter,
+						"pod=%s resource=%s/%s status=%s\n",
+						pod.Name, resourceNamespace, resourceName, pod.Status.Phase,
+					)
+
+					return pod.Status.Phase == corev1.PodRunning
+				}
+			}
+
+			return false
+		}).
+			WithTimeout(time.Minute).
+			WithPolling(time.Second * 5).
+			Should(BeTrue())
+	})
+
+	It("regenerates the bootstrap secret after a restart", func() {
+		ctx := context.Background()
+
+		cluster := v1alpha1.Cluster{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "mycluster",
+				Namespace: namespace,
+			},
+			Spec: v1alpha1.ClusterSpec{
+				TLSSANs: []string{hostIP},
+				Expose: &v1alpha1.ExposeConfig{
+					NodePort: &v1alpha1.NodePortConfig{},
+				},
+				Persistence: v1alpha1.PersistenceConfig{
+					Type: v1alpha1.EphemeralNodeType,
+				},
+			},
+		}
+
+		By(fmt.Sprintf("Creating virtual cluster %s/%s", cluster.Namespace, cluster.Name))
+		NewVirtualCluster(cluster)
+
+		By("Waiting to get a kubernetes client for the virtual cluster")
+		virtualK8sClient := NewVirtualK8sClient(cluster)
+
+		_, err := virtualK8sClient.DiscoveryClient.ServerVersion()
+		Expect(err).To(Not(HaveOccurred()))
+
+		labelSelector := "cluster=" + cluster.Name + ",role=server"
+		serverPods, err := k8s.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+		Expect(err).To(Not(HaveOccurred()))
+
+		Expect(len(serverPods.Items)).To(Equal(1))
+		serverPod := serverPods.Items[0]
+
+		fmt.Fprintf(GinkgoWriter, "deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
+		err = k8s.CoreV1().Pods(namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+
+		By("Deleting server pod")
+
+		// check that the server pods restarted
+		Eventually(func() any {
+			serverPods, err = k8s.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(len(serverPods.Items)).To(Equal(1))
+			return serverPods.Items[0].DeletionTimestamp
+		}).
+			WithTimeout(time.Minute).
+			WithPolling(time.Second * 5).
+			Should(BeNil())
+
+		By("Server pod up and running again")
+
+		By("Using old k8s client configuration should fail")
+
+		Eventually(func() bool {
+			_, err = virtualK8sClient.DiscoveryClient.ServerVersion()
+			var unknownAuthorityErr x509.UnknownAuthorityError
+			return errors.As(err, &unknownAuthorityErr)
+		}).
+			WithTimeout(time.Minute * 2).
+			WithPolling(time.Second * 5).
+			Should(BeTrue())
+
+		By("Recover new config should succeed")
+
+		Eventually(func() error {
+			virtualK8sClient = NewVirtualK8sClient(cluster)
+			_, err = virtualK8sClient.DiscoveryClient.ServerVersion()
+			return err
+		}).
+			WithTimeout(time.Minute * 2).
+			WithPolling(time.Second * 5).
+			Should(BeNil())
+	})
+})
+
+var _ = When("a dynamic cluster is installed", func() {
+
+	var namespace string
+
+	BeforeEach(func() {
+		createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
+		createdNS, err := k8s.CoreV1().Namespaces().Create(context.Background(), createdNS, v1.CreateOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+		namespace = createdNS.Name
+	})
+
+	It("can create a nginx pod", func() {
+		ctx := context.Background()
+
+		cluster := v1alpha1.Cluster{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "mycluster",
+				Namespace: namespace,
+			},
+			Spec: v1alpha1.ClusterSpec{
+				TLSSANs: []string{hostIP},
+				Expose: &v1alpha1.ExposeConfig{
+					NodePort: &v1alpha1.NodePortConfig{},
+				},
+				Persistence: v1alpha1.PersistenceConfig{
+					Type: v1alpha1.DynamicNodesType,
+				},
+			},
+		}
+
+		By(fmt.Sprintf("Creating virtual cluster %s/%s", cluster.Namespace, cluster.Name))
+		NewVirtualCluster(cluster)
+
+		By("Waiting to get a kubernetes client for the virtual cluster")
+		virtualK8sClient := NewVirtualK8sClient(cluster)
+
+		nginxPod := &corev1.Pod{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "nginx",
+				Namespace: "default",
+			},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{{
+					Name:  "nginx",
+					Image: "nginx",
+				}},
+			},
+		}
+		nginxPod, err := virtualK8sClient.CoreV1().Pods(nginxPod.Namespace).Create(ctx, nginxPod, v1.CreateOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+
+		// check that the nginx Pod is up and running in the host cluster
+		Eventually(func() bool {
+			//labelSelector := fmt.Sprintf("%s=%s", translate.ClusterNameLabel, cluster.Namespace)
+			podList, err := k8s.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{})
+			Expect(err).To(Not(HaveOccurred()))
+
+			for _, pod := range podList.Items {
+				resourceName := pod.Annotations[translate.ResourceNameAnnotation]
+				resourceNamespace := pod.Annotations[translate.ResourceNamespaceAnnotation]
+
+				if resourceName == nginxPod.Name && resourceNamespace == nginxPod.Namespace {
+					fmt.Fprintf(GinkgoWriter,
+						"pod=%s resource=%s/%s status=%s\n",
+						pod.Name, resourceNamespace, resourceName, pod.Status.Phase,
+					)
+
+					return pod.Status.Phase == corev1.PodRunning
+				}
+			}
+
+			return false
+		}).
+			WithTimeout(time.Minute).
+			WithPolling(time.Second * 5).
+			Should(BeTrue())
+	})
+
+	It("use the same bootstrap secret after a restart", func() {
+		ctx := context.Background()
+
+		cluster := v1alpha1.Cluster{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "mycluster",
+				Namespace: namespace,
+			},
+			Spec: v1alpha1.ClusterSpec{
+				TLSSANs: []string{hostIP},
+				Expose: &v1alpha1.ExposeConfig{
+					NodePort: &v1alpha1.NodePortConfig{},
+				},
+				Persistence: v1alpha1.PersistenceConfig{
+					Type: v1alpha1.DynamicNodesType,
+				},
+			},
+		}
+
+		By(fmt.Sprintf("Creating virtual cluster %s/%s", cluster.Namespace, cluster.Name))
+		NewVirtualCluster(cluster)
+
+		By("Waiting to get a kubernetes client for the virtual cluster")
+		virtualK8sClient := NewVirtualK8sClient(cluster)
+
+		_, err := virtualK8sClient.DiscoveryClient.ServerVersion()
+		Expect(err).To(Not(HaveOccurred()))
+
+		labelSelector := "cluster=" + cluster.Name + ",role=server"
+		serverPods, err := k8s.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+		Expect(err).To(Not(HaveOccurred()))
+
+		Expect(len(serverPods.Items)).To(Equal(1))
+		serverPod := serverPods.Items[0]
+
+		fmt.Fprintf(GinkgoWriter, "deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
+		err = k8s.CoreV1().Pods(namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+
+		By("Deleting server pod")
+
+		// check that the server pods restarted
+		Eventually(func() any {
+			serverPods, err = k8s.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(len(serverPods.Items)).To(Equal(1))
+			return serverPods.Items[0].DeletionTimestamp
+		}).
+			WithTimeout(30 * time.Second).
+			WithPolling(time.Second * 5).
+			Should(BeNil())
+
+		By("Server pod up and running again")
+
+		By("Using old k8s client configuration should succeed")
+
+		Eventually(func() error {
+			virtualK8sClient = NewVirtualK8sClient(cluster)
+			_, err = virtualK8sClient.DiscoveryClient.ServerVersion()
+			return err
+		}).
+			WithTimeout(2 * time.Minute).
+			WithPolling(time.Second * 5).
+			Should(BeNil())
+	})
+})

tests/common_test.go

@@ -0,0 +1,89 @@
+package k3k_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller/certs"
+	"github.com/rancher/k3k/pkg/controller/kubeconfig"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func NewVirtualCluster(cluster v1alpha1.Cluster) {
+	GinkgoHelper()
+
+	ctx := context.Background()
+	err := k8sClient.Create(ctx, &cluster)
+	Expect(err).To(Not(HaveOccurred()))
+
+	// check that the server Pod and the Kubelet are in Ready state
+	Eventually(func() bool {
+		podList, err := k8s.CoreV1().Pods(cluster.Namespace).List(ctx, v1.ListOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+
+		serverRunning := false
+		kubeletRunning := false
+
+		for _, pod := range podList.Items {
+			imageName := pod.Spec.Containers[0].Image
+			imageName = strings.Split(imageName, ":")[0] // remove tag
+
+			switch imageName {
+			case "rancher/k3s":
+				serverRunning = pod.Status.Phase == corev1.PodRunning
+			case "rancher/k3k-kubelet":
+				kubeletRunning = pod.Status.Phase == corev1.PodRunning
+			}
+
+			if serverRunning && kubeletRunning {
+				return true
+			}
+		}
+
+		return false
+	}).
+		WithTimeout(time.Minute * 2).
+		WithPolling(time.Second * 5).
+		Should(BeTrue())
+
+}
+
+// NewVirtualK8sClient returns a Kubernetes ClientSet for the virtual cluster
+func NewVirtualK8sClient(cluster v1alpha1.Cluster) *kubernetes.Clientset {
+	GinkgoHelper()
+
+	var err error
+	ctx := context.Background()
+
+	var config *clientcmdapi.Config
+	Eventually(func() error {
+		vKubeconfig := kubeconfig.New()
+		kubeletAltName := fmt.Sprintf("k3k-%s-kubelet", cluster.Name)
+		vKubeconfig.AltNames = certs.AddSANs([]string{hostIP, kubeletAltName})
+		config, err = vKubeconfig.Extract(ctx, k8sClient, &cluster, hostIP)
+		return err
+	}).
+		WithTimeout(time.Minute * 2).
+		WithPolling(time.Second * 5).
+		Should(BeNil())
+
+	configData, err := clientcmd.Write(*config)
+	Expect(err).To(Not(HaveOccurred()))
+
+	restcfg, err := clientcmd.RESTConfigFromKubeConfig(configData)
+	Expect(err).To(Not(HaveOccurred()))
+	virtualK8sClient, err := kubernetes.NewForConfig(restcfg)
+	Expect(err).To(Not(HaveOccurred()))
+
+	return virtualK8sClient
+}

tests/k8s_restclientgetter_test.go

@@ -0,0 +1,55 @@
+package k3k_test
+
+import (
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/client-go/discovery"
+	memory "k8s.io/client-go/discovery/cached"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/restmapper"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+type RESTClientGetter struct {
+	clientconfig    clientcmd.ClientConfig
+	restConfig      *rest.Config
+	discoveryClient discovery.CachedDiscoveryInterface
+}
+
+func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) {
+	clientconfig, err := clientcmd.NewClientConfigFromBytes([]byte(kubeconfig))
+	if err != nil {
+		return nil, err
+	}
+
+	restConfig, err := clientconfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	dc, err := discovery.NewDiscoveryClientForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return &RESTClientGetter{
+		clientconfig:    clientconfig,
+		restConfig:      restConfig,
+		discoveryClient: memory.NewMemCacheClient(dc),
+	}, nil
+}
+
+func (r *RESTClientGetter) ToRESTConfig() (*rest.Config, error) {
+	return r.restConfig, nil
+}
+
+func (r *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
+	return r.discoveryClient, nil
+}
+
+func (r *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) {
+	return restmapper.NewDeferredDiscoveryRESTMapper(r.discoveryClient), nil
+}
+
+func (r *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig {
+	return r.clientconfig
+}

tests/tests_suite_test.go

@@ -0,0 +1,186 @@
+package k3k_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"maps"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/go-logr/zapr"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/modules/k3s"
+	"go.uber.org/zap"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+func TestTests(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Tests Suite")
+}
+
+var (
+	k3sContainer *k3s.K3sContainer
+	hostIP       string
+	k8s          *kubernetes.Clientset
+	k8sClient    client.Client
+)
+
+var _ = BeforeSuite(func() {
+	var err error
+	ctx := context.Background()
+
+	k3sContainer, err = k3s.Run(ctx, "rancher/k3s:v1.32.1-k3s1")
+	Expect(err).To(Not(HaveOccurred()))
+
+	hostIP, err = k3sContainer.ContainerIP(ctx)
+	Expect(err).To(Not(HaveOccurred()))
+	fmt.Fprintln(GinkgoWriter, "K3s containerIP: "+hostIP)
+
+	kubeconfig, err := k3sContainer.GetKubeConfig(context.Background())
+	Expect(err).To(Not(HaveOccurred()))
+
+	initKubernetesClient(kubeconfig)
+	installK3kChart(kubeconfig)
+})
+
+func initKubernetesClient(kubeconfig []byte) {
+	restcfg, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig)
+	Expect(err).To(Not(HaveOccurred()))
+
+	k8s, err = kubernetes.NewForConfig(restcfg)
+	Expect(err).To(Not(HaveOccurred()))
+
+	scheme := buildScheme()
+	k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme})
+	Expect(err).NotTo(HaveOccurred())
+
+	logger, err := zap.NewDevelopment()
+	Expect(err).NotTo(HaveOccurred())
+	log.SetLogger(zapr.NewLogger(logger))
+}
+
+func installK3kChart(kubeconfig []byte) {
+	pwd, err := os.Getwd()
+	Expect(err).To(Not(HaveOccurred()))
+
+	k3kChart, err := loader.Load(path.Join(pwd, "../charts/k3k"))
+	Expect(err).To(Not(HaveOccurred()))
+
+	actionConfig := new(action.Configuration)
+
+	restClientGetter, err := NewRESTClientGetter(kubeconfig)
+	Expect(err).To(Not(HaveOccurred()))
+
+	releaseName := "k3k"
+	releaseNamespace := "k3k-system"
+
+	err = actionConfig.Init(restClientGetter, releaseNamespace, os.Getenv("HELM_DRIVER"), func(format string, v ...interface{}) {
+		fmt.Fprintf(GinkgoWriter, "helm debug: "+format+"\n", v...)
+	})
+	Expect(err).To(Not(HaveOccurred()))
+
+	iCli := action.NewInstall(actionConfig)
+	iCli.ReleaseName = releaseName
+	iCli.Namespace = releaseNamespace
+	iCli.CreateNamespace = true
+	iCli.Timeout = time.Minute
+	iCli.Wait = true
+
+	imageMap, _ := k3kChart.Values["image"].(map[string]any)
+	maps.Copy(imageMap, map[string]any{
+		"repository": "rancher/k3k",
+		"tag":        "dev",
+		"pullPolicy": "IfNotPresent",
+	})
+
+	sharedAgentMap, _ := k3kChart.Values["sharedAgent"].(map[string]any)
+	sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any)
+	maps.Copy(sharedAgentImageMap, map[string]any{
+		"repository": "rancher/k3k-kubelet",
+		"tag":        "dev",
+	})
+
+	err = k3sContainer.LoadImages(context.Background(), "rancher/k3k:dev", "rancher/k3k-kubelet:dev")
+	Expect(err).To(Not(HaveOccurred()))
+
+	release, err := iCli.Run(k3kChart, k3kChart.Values)
+	Expect(err).To(Not(HaveOccurred()))
+
+	fmt.Fprintf(GinkgoWriter, "Release %s installed in %s namespace\n", release.Name, release.Namespace)
+}
+
+var _ = AfterSuite(func() {
+	// dump k3s logs
+	readCloser, err := k3sContainer.Logs(context.Background())
+	Expect(err).To(Not(HaveOccurred()))
+
+	logs, err := io.ReadAll(readCloser)
+	Expect(err).To(Not(HaveOccurred()))
+
+	logfile := path.Join(os.TempDir(), "k3s.log")
+	err = os.WriteFile(logfile, logs, 0644)
+	Expect(err).To(Not(HaveOccurred()))
+
+	fmt.Fprintln(GinkgoWriter, "k3s logs written to: "+logfile)
+
+	// dump k3k controller logs
+	readCloser, err = k3sContainer.Logs(context.Background())
+	Expect(err).To(Not(HaveOccurred()))
+	writeLogs("k3s.log", readCloser)
+
+	// dump k3k logs
+	writeK3kLogs()
+
+	testcontainers.CleanupContainer(GinkgoTB(), k3sContainer)
+})
+
+func buildScheme() *runtime.Scheme {
+	scheme := runtime.NewScheme()
+
+	err := corev1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+	err = v1alpha1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	return scheme
+}
+
+func writeK3kLogs() {
+	var err error
+	var podList v1.PodList
+
+	ctx := context.Background()
+	err = k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: "k3k-system"})
+	Expect(err).To(Not(HaveOccurred()))
+
+	k3kPod := podList.Items[0]
+	req := k8s.CoreV1().Pods(k3kPod.Namespace).GetLogs(k3kPod.Name, &corev1.PodLogOptions{})
+	podLogs, err := req.Stream(ctx)
+	Expect(err).To(Not(HaveOccurred()))
+	writeLogs("k3k.log", podLogs)
+}
+
+func writeLogs(filename string, logs io.ReadCloser) {
+	logsStr, err := io.ReadAll(logs)
+	Expect(err).To(Not(HaveOccurred()))
+	defer logs.Close()
+	tempfile := path.Join(os.TempDir(), filename)
+	err = os.WriteFile(tempfile, []byte(logsStr), 0644)
+	Expect(err).To(Not(HaveOccurred()))
+	fmt.Fprintln(GinkgoWriter, "logs written to: "+filename)
+}