bump version to 1.0.1-rc1 in Chart.yaml (#567)

* added --namespace flag to policy create to actually bind the new policy to existing namespaces

* fix lint

* fix tests

* added overwrite flag

* updated cli docs

* fix tests 2

* moved double quotes to single quote

* fix test

.github/workflows/build.yml

@@ -13,6 +13,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    permissions:
+      contents: read
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
@@ -34,4 +38,51 @@ jobs:
       env:
         REPO: ${{ github.repository }}
         REGISTRY: ""
-        
+
+    - name: Run Trivy vulnerability scanner (k3kcli)
+      uses: aquasecurity/trivy-action@0.28.0
+      with:
+        ignore-unfixed: true
+        severity: 'MEDIUM,HIGH,CRITICAL'
+        scan-type: 'fs'
+        scan-ref: 'dist/k3kcli_linux_amd64_v1/k3kcli'
+        format: 'sarif'
+        output: 'trivy-results-k3kcli.sarif'
+
+    - name: Upload Trivy scan results to GitHub Security tab (k3kcli)
+      uses: github/codeql-action/upload-sarif@v3
+      with:
+        sarif_file: trivy-results-k3kcli.sarif
+        category: k3kcli
+
+    - name: Run Trivy vulnerability scanner (k3k)
+      uses: aquasecurity/trivy-action@0.28.0
+      with:
+        ignore-unfixed: true
+        severity: 'MEDIUM,HIGH,CRITICAL'
+        scan-type: 'image'
+        scan-ref: '${{ github.repository }}:v0.0.0-amd64'
+        format: 'sarif'
+        output: 'trivy-results-k3k.sarif'
+
+    - name: Upload Trivy scan results to GitHub Security tab (k3k)
+      uses: github/codeql-action/upload-sarif@v3
+      with:
+        sarif_file: trivy-results-k3k.sarif
+        category: k3k
+
+    - name: Run Trivy vulnerability scanner (k3k-kubelet)
+      uses: aquasecurity/trivy-action@0.28.0
+      with:
+        ignore-unfixed: true
+        severity: 'MEDIUM,HIGH,CRITICAL'
+        scan-type: 'image'
+        scan-ref: '${{ github.repository }}-kubelet:v0.0.0-amd64'
+        format: 'sarif'
+        output: 'trivy-results-k3k-kubelet.sarif'
+
+    - name: Upload Trivy scan results to GitHub Security tab (k3k-kubelet)
+      uses: github/codeql-action/upload-sarif@v3
+      with:
+        sarif_file: trivy-results-k3k-kubelet.sarif
+        category: k3k-kubelet

.github/workflows/test-conformance-shared.yaml

@@ -0,0 +1,158 @@
+name: Conformance Tests - Shared Mode
+
+on:
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+
+permissions:
+    contents: read
+
+jobs:
+  conformance:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        type:
+        - parallel
+        - serial
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Install helm
+      uses: azure/setup-helm@v4.3.0
+    
+    - name: Install hydrophone
+      run: go install sigs.k8s.io/hydrophone@latest
+
+    - name: Install k3d and kubectl
+      run: |
+        wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+        k3d version
+
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+
+    - name: Setup Kubernetes (k3d)
+      env:
+        REPO_NAME: k3k-registry
+        REPO_PORT: 12345
+      run: |
+        echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts
+
+        k3d registry create ${REPO_NAME} --port ${REPO_PORT}
+        
+        k3d cluster create k3k --servers 2 \
+          -p "30000-30010:30000-30010@server:0" \
+          --registry-use k3d-${REPO_NAME}:${REPO_PORT}
+        
+        kubectl cluster-info
+        kubectl get nodes
+
+    - name: Setup K3k
+      env:
+        REPO: k3k-registry:12345
+      run: |
+        echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts
+
+        make build
+        make package
+        make push
+
+        # add k3kcli to $PATH
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+        VERSION=$(make version)
+        k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose
+        k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose
+
+        make install
+        
+        echo "Wait for K3k controller to be available"
+        kubectl wait -n k3k-system pod --for condition=Ready -l "app.kubernetes.io/name=k3k" --timeout=5m
+
+    - name: Check k3kcli
+      run: k3kcli -v
+
+    - name: Create virtual cluster
+      run: |
+        kubectl create namespace k3k-mycluster
+
+        cat <<EOF | kubectl apply -f -
+        apiVersion: k3k.io/v1beta1
+        kind: Cluster
+        metadata:
+          name: mycluster
+          namespace: k3k-mycluster
+        spec:
+          mirrorHostNodes: true
+          tlsSANs:
+            - "127.0.0.1"
+          expose:
+            nodePort:
+              serverPort: 30001
+        EOF
+
+        echo "Wait for bootstrap secret to be available"
+        kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m
+
+        k3kcli kubeconfig generate --name mycluster
+
+        export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
+        
+        kubectl cluster-info
+        kubectl get nodes
+        kubectl get pods -A
+    
+    - name: Run conformance tests (parallel)
+      if: matrix.type == 'parallel'
+      run: |
+        # Run conformance tests in parallel mode (skipping serial)
+        hydrophone --conformance --parallel 4 --skip='\[Serial\]' \
+          --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
+          --output-dir /tmp
+
+    - name: Run conformance tests (serial)
+      if: matrix.type == 'serial'
+      run: |        
+        # Run serial conformance tests
+        hydrophone --focus='\[Serial\].*\[Conformance\]' \
+          --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
+          --output-dir /tmp 
+
+    - name: Archive conformance logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: conformance-${{ matrix.type }}-logs
+        path: /tmp/e2e.log
+
+    - name: Job Summary
+      if: always()
+      run: |
+        echo '## 📊 Conformance Tests Results (${{ matrix.type }})' >> $GITHUB_STEP_SUMMARY
+        echo '| Passed | Failed | Pending | Skipped |'              >> $GITHUB_STEP_SUMMARY
+        echo '|---|---|---|---|'                                    >> $GITHUB_STEP_SUMMARY
+
+        RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3)
+        RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g')
+        echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY
+
+        # only include failed tests section if there are any
+        if grep -q '\[FAIL\]' /tmp/e2e.log; then
+          echo ''                       >> $GITHUB_STEP_SUMMARY
+          echo '### Failed Tests'       >> $GITHUB_STEP_SUMMARY
+          echo '```'                    >> $GITHUB_STEP_SUMMARY
+          grep '\[FAIL\]' /tmp/e2e.log  >> $GITHUB_STEP_SUMMARY
+          echo '```'                    >> $GITHUB_STEP_SUMMARY
+        fi

.github/workflows/test-conformance-virtual.yaml

@@ -0,0 +1,145 @@
+name: Conformance Tests - Virtual Mode
+
+on:
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+
+permissions:
+    contents: read
+
+jobs:
+  conformance:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        type:
+        - parallel
+        - serial
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Install helm
+      uses: azure/setup-helm@v4.3.0
+    
+    - name: Install hydrophone
+      run: go install sigs.k8s.io/hydrophone@latest
+
+    - name: Install k3s
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+        K3S_HOST_VERSION: v1.32.1+k3s1
+      run: |
+        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
+
+        kubectl cluster-info
+        kubectl get nodes
+
+    - name: Build, package and setup K3k
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      run: |
+        export REPO=ttl.sh/$(uuidgen)
+        export VERSION=1h
+
+        make build
+        make package
+        make push
+        make install
+
+        # add k3kcli to $PATH
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+        
+        echo "Wait for K3k controller to be available"
+        kubectl wait -n k3k-system pod --for condition=Ready -l "app.kubernetes.io/name=k3k" --timeout=5m
+
+    - name: Check k3kcli
+      run: k3kcli -v
+
+    - name: Create virtual cluster
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      run: |
+        k3kcli cluster create --mode=virtual --servers=2 mycluster
+
+        export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml        
+        
+        kubectl cluster-info
+        kubectl get nodes
+        kubectl get pods -A
+    
+    - name: Run conformance tests (parallel)
+      if: matrix.type == 'parallel'
+      run: |
+        # Run conformance tests in parallel mode (skipping serial)
+        hydrophone --conformance --parallel 4 --skip='\[Serial\]' \
+          --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
+          --output-dir /tmp
+
+    - name: Run conformance tests (serial)
+      if: matrix.type == 'serial'
+      run: |        
+        # Run serial conformance tests
+        hydrophone --focus='\[Serial\].*\[Conformance\]' \
+          --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
+          --output-dir /tmp
+    
+    - name: Export logs
+      if: always()
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      run: |
+        journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
+        kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log
+
+    - name: Archive K3s logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: k3s-${{ matrix.type }}-logs
+        path: /tmp/k3s.log
+
+    - name: Archive K3k logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: k3k-${{ matrix.type }}-logs
+        path: /tmp/k3k.log
+
+    - name: Archive conformance logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: conformance-${{ matrix.type }}-logs
+        path: /tmp/e2e.log
+
+    - name: Job Summary
+      if: always()
+      run: |
+        echo '## 📊 Conformance Tests Results (${{ matrix.type }})' >> $GITHUB_STEP_SUMMARY
+        echo '| Passed | Failed | Pending | Skipped |'              >> $GITHUB_STEP_SUMMARY
+        echo '|---|---|---|---|'                                    >> $GITHUB_STEP_SUMMARY
+
+        RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3)
+        RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g')
+        echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY
+
+        # only include failed tests section if there are any
+        if grep -q '\[FAIL\]' /tmp/e2e.log; then
+          echo ''                       >> $GITHUB_STEP_SUMMARY
+          echo '### Failed Tests'       >> $GITHUB_STEP_SUMMARY
+          echo '```'                    >> $GITHUB_STEP_SUMMARY
+          grep '\[FAIL\]' /tmp/e2e.log  >> $GITHUB_STEP_SUMMARY
+          echo '```'                    >> $GITHUB_STEP_SUMMARY
+        fi

.github/workflows/test-conformance.yaml

@@ -1,302 +0,0 @@
-name: Conformance Tests
-
-on:
-  schedule:
-    - cron: "0 1 * * *"
-  workflow_dispatch:
-    inputs:
-      test:
-        description: "Run specific test"
-        type: choice
-        options:
-          - conformance
-          - sig-api-machinery
-          - sig-apps
-          - sig-architecture
-          - sig-auth
-          - sig-cli
-          - sig-instrumentation
-          - sig-network
-          - sig-node
-          - sig-scheduling
-          - sig-storage
-
-permissions:
-    contents: read
-
-jobs:
-  conformance:
-    runs-on: ubuntu-latest
-    if: inputs.test == '' || inputs.test == 'conformance'
-
-    strategy:
-      fail-fast: false
-      matrix:
-        type:
-        - parallel
-        - serial
-
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        fetch-tags: true
-
-    - uses: actions/setup-go@v5
-      with:
-        go-version-file: go.mod
-
-    - name: Install helm
-      uses: azure/setup-helm@v4.3.0
-    
-    - name: Install hydrophone
-      run: go install sigs.k8s.io/hydrophone@latest
-
-    - name: Install k3d and kubectl
-      run: |
-        wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
-        k3d version
-
-        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
-
-    - name: Setup Kubernetes (k3d)
-      env:
-        REPO_NAME: k3k-registry
-        REPO_PORT: 12345
-      run: |
-        echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts
-
-        k3d registry create ${REPO_NAME} --port ${REPO_PORT}
-        
-        k3d cluster create k3k --servers 3 \
-          -p "30000-30010:30000-30010@server:0" \
-          --registry-use k3d-${REPO_NAME}:${REPO_PORT}
-        
-        kubectl cluster-info
-        kubectl get nodes
-
-    - name: Setup K3k
-      env:
-        REPO: k3k-registry:12345
-      run: |
-        echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts
-
-        make build
-        make package
-        make push
-
-        # add k3kcli to $PATH
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-
-        VERSION=$(make version)
-        k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose
-        k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose
-
-        make install
-        
-        echo "Wait for K3k controller to be available"
-        kubectl wait -n k3k-system pod --for condition=Ready -l "app.kubernetes.io/name=k3k" --timeout=5m
-
-    - name: Check k3kcli
-      run: k3kcli -v
-
-    - name: Create virtual cluster
-      run: |
-        kubectl create namespace k3k-mycluster
-
-        cat <<EOF | kubectl apply -f -
-        apiVersion: k3k.io/v1beta1
-        kind: Cluster
-        metadata:
-          name: mycluster
-          namespace: k3k-mycluster
-        spec:
-          servers: 2
-          mirrorHostNodes: true
-          tlsSANs:
-            - "127.0.0.1"
-          expose:
-            nodePort:
-              serverPort: 30001
-        EOF
-
-        echo "Wait for bootstrap secret to be available"
-        kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m
-
-        k3kcli kubeconfig generate --name mycluster
-
-        export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
-        
-        kubectl cluster-info
-        kubectl get nodes
-        kubectl get pods -A
-    
-    - name: Run conformance tests (parallel)
-      if: matrix.type == 'parallel'
-      run: |
-        # Run conformance tests in parallel mode (skipping serial)
-        hydrophone --conformance --parallel 4 --skip='\[Serial\]' \
-          --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
-          --output-dir /tmp
-
-    - name: Run conformance tests (serial)
-      if: matrix.type == 'serial'
-      run: |        
-        # Run serial conformance tests
-        hydrophone --focus='\[Serial\].*\[Conformance\]' \
-          --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
-          --output-dir /tmp 
-
-    - name: Archive conformance logs
-      uses: actions/upload-artifact@v4
-      if: always()
-      with:
-        name: conformance-${{ matrix.type }}-logs
-        path: /tmp/e2e.log
-
-  sigs:
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        tests:
-          - name: sig-api-machinery
-            focus: '\[sig-api-machinery\].*\[Conformance\]'
-          - name: sig-apps
-            focus: '\[sig-apps\].*\[Conformance\]'
-          - name: sig-architecture
-            focus: '\[sig-architecture\].*\[Conformance\]'
-          - name: sig-auth
-            focus: '\[sig-auth\].*\[Conformance\]'
-          - name: sig-cli
-            focus: '\[sig-cli\].*\[Conformance\]'
-          - name: sig-instrumentation
-            focus: '\[sig-instrumentation\].*\[Conformance\]'
-          - name: sig-network
-            focus: '\[sig-network\].*\[Conformance\]'
-          - name: sig-node
-            focus: '\[sig-node\].*\[Conformance\]'
-          - name: sig-scheduling
-            focus: '\[sig-scheduling\].*\[Conformance\]'
-          - name: sig-storage
-            focus: '\[sig-storage\].*\[Conformance\]'
-
-    steps:
-    - name: Validate input and fail fast
-      if: inputs.test != '' && inputs.test != matrix.tests.name
-      run: |
-        echo "Failing this job as it's not the intended target."
-        exit 1
-
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        fetch-tags: true
-
-    - uses: actions/setup-go@v5
-      with:
-        go-version-file: go.mod
-
-    - name: Install helm
-      uses: azure/setup-helm@v4.3.0
-    
-    - name: Install hydrophone
-      run: go install sigs.k8s.io/hydrophone@latest
-
-    - name: Install k3d and kubectl
-      run: |
-        wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
-        k3d version
-
-        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
-
-    - name: Setup Kubernetes (k3d)
-      env:
-        REPO_NAME: k3k-registry
-        REPO_PORT: 12345
-      run: |
-        echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts
-
-        k3d registry create ${REPO_NAME} --port ${REPO_PORT}
-        
-        k3d cluster create k3k --servers 3 \
-          -p "30000-30010:30000-30010@server:0" \
-          --registry-use k3d-${REPO_NAME}:${REPO_PORT}
-        
-        kubectl cluster-info
-        kubectl get nodes
-
-    - name: Setup K3k
-      env:
-        REPO: k3k-registry:12345
-      run: |
-        echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts
-
-        make build
-        make package
-        make push
-
-        # add k3kcli to $PATH
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-
-        VERSION=$(make version)
-        k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose
-        k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose
-
-        make install
-        
-        echo "Wait for K3k controller to be available"
-        kubectl wait -n k3k-system pod --for condition=Ready -l "app.kubernetes.io/name=k3k" --timeout=5m
-
-    - name: Check k3kcli
-      run: k3kcli -v
-
-    - name: Create virtual cluster
-      run: |
-        kubectl create namespace k3k-mycluster
-
-        cat <<EOF | kubectl apply -f -
-        apiVersion: k3k.io/v1beta1
-        kind: Cluster
-        metadata:
-          name: mycluster
-          namespace: k3k-mycluster
-        spec:
-          servers: 2
-          mirrorHostNodes: true
-          tlsSANs:
-            - "127.0.0.1"
-          expose:
-            nodePort:
-              serverPort: 30001
-        EOF
-
-        echo "Wait for bootstrap secret to be available"
-        kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m
-
-        k3kcli kubeconfig generate --name mycluster
-
-        export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
-        
-        kubectl cluster-info
-        kubectl get nodes
-        kubectl get pods -A
-
-    - name: Run sigs tests
-      run: |
-        FOCUS="${{ matrix.tests.focus }}"
-        echo "Running with --focus=${FOCUS}"
-
-        hydrophone --focus "${FOCUS}" \
-          --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
-          --output-dir /tmp
-
-    - name: Archive conformance logs
-      uses: actions/upload-artifact@v4
-      if: always()
-      with:
-        name: ${{ matrix.tests.name }}-logs
-        path: /tmp/e2e.log

.github/workflows/test-e2e.yaml

@@ -0,0 +1,184 @@
+name: Tests E2E
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+    contents: read
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Validate
+      run: make validate
+  tests-e2e:
+    runs-on: ubuntu-latest
+    needs: validate
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Install Ginkgo
+      run: go install github.com/onsi/ginkgo/v2/ginkgo
+    
+    - name: Setup environment
+      run: |
+        mkdir ${{ github.workspace }}/covdata
+        
+        echo "COVERAGE=true" >> $GITHUB_ENV
+        echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV
+        echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV
+        echo "VERSION=1h" >> $GITHUB_ENV
+        echo "K3S_HOST_VERSION=v1.32.1+k3s1 >> $GITHUB_ENV"
+
+    - name: Install k3s
+      run: |
+        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ env.K3S_HOST_VERSION }} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - 
+
+    - name: Build and package and push dev images
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+        REPO: ${{ env.REPO }}
+        VERSION: ${{ env.VERSION }}
+      run: |
+        make build
+        make package
+        make push
+        make install
+
+    - name: Run e2e tests
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+        REPO: ${{ env.REPO }}
+        VERSION: ${{ env.VERSION }}
+      run:  make E2E_LABEL_FILTER="e2e && !slow" test-e2e
+
+    - name: Convert coverage data
+      run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
+    
+    - name: Upload coverage reports to Codecov (controller)
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ${GOCOVERDIR}/cover.out
+        flags: controller
+
+    - name: Upload coverage reports to Codecov (e2e)
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./cover.out
+        flags: e2e
+
+    - name: Archive k3s logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: e2e-k3s-logs
+        path: /tmp/k3s.log
+
+    - name: Archive k3k logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: e2e-k3k-logs
+        path: /tmp/k3k.log
+  tests-e2e-slow:
+    runs-on: ubuntu-latest
+    needs: validate
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Install Ginkgo
+      run: go install github.com/onsi/ginkgo/v2/ginkgo
+    
+    - name: Setup environment
+      run: |
+        mkdir ${{ github.workspace }}/covdata
+        
+        echo "COVERAGE=true" >> $GITHUB_ENV
+        echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV
+        echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV
+        echo "VERSION=1h" >> $GITHUB_ENV
+        echo "K3S_HOST_VERSION=v1.32.1+k3s1 >> $GITHUB_ENV"
+
+    - name: Install k3s
+      run: |
+        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ env.K3S_HOST_VERSION }} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - 
+
+    - name: Build and package and push dev images
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+        REPO: ${{ env.REPO }}
+        VERSION: ${{ env.VERSION }}
+      run: |
+        make build
+        make package
+        make push
+        make install
+
+    - name: Run e2e tests
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+        REPO: ${{ env.REPO }}
+        VERSION: ${{ env.VERSION }}
+      run:  make E2E_LABEL_FILTER="e2e && slow" test-e2e
+
+    - name: Convert coverage data
+      run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
+    
+    - name: Upload coverage reports to Codecov (controller)
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ${GOCOVERDIR}/cover.out
+        flags: controller
+
+    - name: Upload coverage reports to Codecov (e2e)
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./cover.out
+        flags: e2e
+
+    - name: Archive k3s logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: e2e-k3s-logs
+        path: /tmp/k3s.log
+
+    - name: Archive k3k logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: e2e-k3k-logs
+        path: /tmp/k3k.log

.github/workflows/test.yaml

@@ -62,93 +62,6 @@ jobs:
         files: ./cover.out
         flags: unit
 
-  tests-e2e:
-    runs-on: ubuntu-latest
-    needs: validate
-
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        fetch-tags: true
-
-    - uses: actions/setup-go@v5
-      with:
-        go-version-file: go.mod
-
-    - name: Install Ginkgo
-      run: go install github.com/onsi/ginkgo/v2/ginkgo
-    
-    - name: Setup environment
-      run: |
-        mkdir ${{ github.workspace }}/covdata
-        
-        echo "COVERAGE=true" >> $GITHUB_ENV
-        echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV
-        echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV
-        echo "VERSION=1h" >> $GITHUB_ENV
-        echo "K3S_HOST_VERSION=v1.32.1+k3s1 >> $GITHUB_ENV"
-
-    - name: Install k3s
-      run: |
-        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ env.K3S_HOST_VERSION }} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - 
-
-    - name: Build and package and push dev images
-      env:
-        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-        REPO: ${{ env.REPO }}
-        VERSION: ${{ env.VERSION }}
-      run: |
-        make build
-        make package
-        make push
-        make install
-
-        # add k3kcli to $PATH
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-
-    - name: Check k3kcli
-      run: k3kcli -v
-
-    - name: Run e2e tests
-      env:
-        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-        REPO: ${{ env.REPO }}
-        VERSION: ${{ env.VERSION }}
-      run:  make test-e2e
-
-    - name: Convert coverage data
-      run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
-    
-    - name: Upload coverage reports to Codecov (controller)
-      uses: codecov/codecov-action@v5
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        files: ${GOCOVERDIR}/cover.out
-        flags: controller
-
-    - name: Upload coverage reports to Codecov (e2e)
-      uses: codecov/codecov-action@v5
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        files: ./cover.out
-        flags: e2e
-
-    - name: Archive k3s logs
-      uses: actions/upload-artifact@v4
-      if: always()
-      with:
-        name: e2e-k3s-logs
-        path: /tmp/k3s.log
-
-    - name: Archive k3k logs
-      uses: actions/upload-artifact@v4
-      if: always()
-      with:
-        name: e2e-k3k-logs
-        path: /tmp/k3k.log
-
   tests-cli:
     runs-on: ubuntu-latest
     needs: validate

Makefile

@@ -18,6 +18,9 @@ CRD_REF_DOCS := go run github.com/elastic/crd-ref-docs@$(CRD_REF_DOCS_VER)
 
 ENVTEST ?= go run sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
 ENVTEST_DIR ?= $(shell pwd)/.envtest
+
+E2E_LABEL_FILTER ?= "e2e"
+
 export KUBEBUILDER_ASSETS ?= $(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(ENVTEST_DIR) -p path)
 
 
@@ -69,7 +72,7 @@ test-kubelet-controller:	## Run the controller tests (pkg/controller)
 
 .PHONY: test-e2e
 test-e2e:	## Run the e2e tests
-	$(GINKGO) $(GINKGO_FLAGS) --label-filter=e2e tests
+	$(GINKGO) $(GINKGO_FLAGS) --label-filter="$(E2E_LABEL_FILTER)" tests
 
 .PHONY: test-cli
 test-cli:	## Run the cli tests
@@ -105,6 +108,8 @@ validate: generate docs fmt ## Validate the project checking for any dependency
 .PHONY: install
 install:	## Install K3k with Helm on the targeted Kubernetes cluster
 	helm upgrade --install --namespace k3k-system --create-namespace \
+		--set controller.extraEnv[0].name=DEBUG \
+		--set-string controller.extraEnv[0].value=true \
 		--set controller.image.repository=$(REPO)/k3k \
 		--set controller.image.tag=$(VERSION) \
 		--set agent.shared.image.repository=$(REPO)/k3k-kubelet \

README.md

@@ -1,9 +1,9 @@
 # K3k: Kubernetes in Kubernetes
 
-[![Experimental](https://img.shields.io/badge/status-experimental-orange.svg)](https://shields.io/)
 [![Go Report Card](https://goreportcard.com/badge/github.com/rancher/k3k)](https://goreportcard.com/report/github.com/rancher/k3k)
 ![Tests](https://github.com/rancher/k3k/actions/workflows/test.yaml/badge.svg)
-![Build](https://github.com/rancher/k3k/actions/workflows/build.yml/badge.svg)
+![Build](https://github.com/rancher/k3k/actions/workflows/build.yml/badge.svg)  
+[![Conformance Tests - Virtual Mode](https://github.com/rancher/k3k/actions/workflows/test-conformance-virtual.yaml/badge.svg)](https://github.com/rancher/k3k/actions/workflows/test-conformance-virtual.yaml)
 
 
 K3k, Kubernetes in Kubernetes, is a tool that empowers you to create and manage isolated K3s clusters within your existing Kubernetes environment.  It enables efficient multi-tenancy, streamlined experimentation, and robust resource isolation, minimizing infrastructure costs by allowing you to run multiple lightweight Kubernetes clusters on the same physical host. K3k offers both "shared" mode, optimizing resource utilization, and "virtual" mode, providing complete isolation with dedicated K3s server pods. This allows you to access a full Kubernetes experience without the overhead of managing separate physical resources. 
@@ -11,10 +11,6 @@ K3k, Kubernetes in Kubernetes, is a tool that empowers you to create and manage
 K3k integrates seamlessly with Rancher for simplified management of your embedded clusters.
 
 
-**Experimental Tool**
-
-This project is still under development and is considered experimental. It may have limitations, bugs, or changes. Please use with caution and report any issues you encounter. We appreciate your feedback as we continue to refine and improve this tool.
-
 
 ## Features and Benefits
 
@@ -59,7 +55,7 @@ This section provides instructions on how to install K3k and the `k3kcli`.
    helm install --namespace k3k-system --create-namespace k3k k3k/k3k
    ```
 
-**NOTE:** K3k is currently under development. We recommend using the latest released version when possible.
+We recommend using the latest released version when possible.
 
 
 ### Install the `k3kcli`
@@ -71,7 +67,7 @@ To install it, simply download the latest available version for your architectur
 For example, you can download the Linux amd64 version with:
 
 ```
-wget -qO k3kcli https://github.com/rancher/k3k/releases/download/v0.3.5/k3kcli-linux-amd64 && \
+wget -qO k3kcli https://github.com/rancher/k3k/releases/download/v1.0.0/k3kcli-linux-amd64 && \
   chmod +x k3kcli && \
   sudo mv k3kcli /usr/local/bin
 ```
@@ -79,7 +75,7 @@ wget -qO k3kcli https://github.com/rancher/k3k/releases/download/v0.3.5/k3kcli-l
 You should now be able to run:
 ```bash
 -> % k3kcli --version
-k3kcli version v0.3.5
+k3kcli version v1.0.0
 ```
 
 

charts/k3k/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: k3k
 description: A Helm chart for K3K
 type: application
-version: 0.3.5
-appVersion: v0.3.5
+version: 1.0.1-rc1
+appVersion: v1.0.1-rc1

charts/k3k/templates/crds/k3k.io_clusters.yaml

@@ -3,6 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
+    helm.sh/resource-policy: keep
     controller-gen.kubebuilder.io/version: v0.16.0
   name: clusters.k3k.io
 spec:
@@ -228,6 +229,7 @@ spec:
                   certificates.
                 properties:
                   enabled:
+                    default: true
                     description: Enabled toggles this feature on or off.
                     type: boolean
                   sources:
@@ -244,6 +246,8 @@ spec:
                               - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
                               - For ServiceAccountTokenKey: 'tls.key'.
                             type: string
+                        required:
+                        - secretName
                         type: object
                       etcdPeerCA:
                         description: ETCDPeerCA specifies the etcd-peer-ca cert/key
@@ -256,6 +260,8 @@ spec:
                               - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
                               - For ServiceAccountTokenKey: 'tls.key'.
                             type: string
+                        required:
+                        - secretName
                         type: object
                       etcdServerCA:
                         description: ETCDServerCA specifies the etcd-server-ca cert/key
@@ -268,6 +274,8 @@ spec:
                               - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
                               - For ServiceAccountTokenKey: 'tls.key'.
                             type: string
+                        required:
+                        - secretName
                         type: object
                       requestHeaderCA:
                         description: RequestHeaderCA specifies the request-header-ca
@@ -280,6 +288,8 @@ spec:
                               - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
                               - For ServiceAccountTokenKey: 'tls.key'.
                             type: string
+                        required:
+                        - secretName
                         type: object
                       serverCA:
                         description: ServerCA specifies the server-ca cert/key pair.
@@ -291,6 +301,8 @@ spec:
                               - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
                               - For ServiceAccountTokenKey: 'tls.key'.
                             type: string
+                        required:
+                        - secretName
                         type: object
                       serviceAccountToken:
                         description: ServiceAccountToken specifies the service-account-token
@@ -303,8 +315,20 @@ spec:
                               - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
                               - For ServiceAccountTokenKey: 'tls.key'.
                             type: string
+                        required:
+                        - secretName
                         type: object
+                    required:
+                    - clientCA
+                    - etcdPeerCA
+                    - etcdServerCA
+                    - requestHeaderCA
+                    - serverCA
+                    - serviceAccountToken
                     type: object
+                required:
+                - enabled
+                - sources
                 type: object
               expose:
                 description: |-
@@ -326,7 +350,7 @@ spec:
                           use for the Ingress.
                         type: string
                     type: object
-                  loadbalancer:
+                  loadBalancer:
                     description: LoadBalancer specifies options for exposing the API
                       server through a LoadBalancer service.
                     properties:
@@ -368,7 +392,7 @@ spec:
                 x-kubernetes-validations:
                 - message: ingress, loadbalancer and nodePort are mutually exclusive;
                     only one can be set
-                  rule: '[has(self.ingress), has(self.loadbalancer), has(self.nodePort)].filter(x,
+                  rule: '[has(self.ingress), has(self.loadBalancer), has(self.nodePort)].filter(x,
                     x).size() <= 1'
               mirrorHostNodes:
                 description: |-
@@ -584,12 +608,13 @@ spec:
                 description: Sync specifies the resources types that will be synced
                   from virtual cluster to host cluster.
                 properties:
-                  configmaps:
+                  configMaps:
                     default:
                       enabled: true
                     description: ConfigMaps resources sync configuration.
                     properties:
                       enabled:
+                        default: true
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -599,6 +624,8 @@ spec:
                           Selector specifies set of labels of the resources that will be synced, if empty
                           then all resources of the given type will be synced.
                         type: object
+                    required:
+                    - enabled
                     type: object
                   ingresses:
                     default:
@@ -606,6 +633,7 @@ spec:
                     description: Ingresses resources sync configuration.
                     properties:
                       enabled:
+                        default: false
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -615,6 +643,8 @@ spec:
                           Selector specifies set of labels of the resources that will be synced, if empty
                           then all resources of the given type will be synced.
                         type: object
+                    required:
+                    - enabled
                     type: object
                   persistentVolumeClaims:
                     default:
@@ -622,6 +652,7 @@ spec:
                     description: PersistentVolumeClaims resources sync configuration.
                     properties:
                       enabled:
+                        default: true
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -631,6 +662,8 @@ spec:
                           Selector specifies set of labels of the resources that will be synced, if empty
                           then all resources of the given type will be synced.
                         type: object
+                    required:
+                    - enabled
                     type: object
                   priorityClasses:
                     default:
@@ -638,6 +671,7 @@ spec:
                     description: PriorityClasses resources sync configuration.
                     properties:
                       enabled:
+                        default: false
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -647,6 +681,8 @@ spec:
                           Selector specifies set of labels of the resources that will be synced, if empty
                           then all resources of the given type will be synced.
                         type: object
+                    required:
+                    - enabled
                     type: object
                   secrets:
                     default:
@@ -654,6 +690,7 @@ spec:
                     description: Secrets resources sync configuration.
                     properties:
                       enabled:
+                        default: true
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -670,6 +707,7 @@ spec:
                     description: Services resources sync configuration.
                     properties:
                       enabled:
+                        default: true
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -679,6 +717,8 @@ spec:
                           Selector specifies set of labels of the resources that will be synced, if empty
                           then all resources of the given type will be synced.
                         type: object
+                    required:
+                    - enabled
                     type: object
                 type: object
               tlsSANs:

charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml

@@ -3,6 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
+    helm.sh/resource-policy: keep
     controller-gen.kubebuilder.io/version: v0.16.0
   name: virtualclusterpolicies.k3k.io
 spec:
@@ -230,12 +231,13 @@ spec:
                 description: Sync specifies the resources types that will be synced
                   from virtual cluster to host cluster.
                 properties:
-                  configmaps:
+                  configMaps:
                     default:
                       enabled: true
                     description: ConfigMaps resources sync configuration.
                     properties:
                       enabled:
+                        default: true
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -245,6 +247,8 @@ spec:
                           Selector specifies set of labels of the resources that will be synced, if empty
                           then all resources of the given type will be synced.
                         type: object
+                    required:
+                    - enabled
                     type: object
                   ingresses:
                     default:
@@ -252,6 +256,7 @@ spec:
                     description: Ingresses resources sync configuration.
                     properties:
                       enabled:
+                        default: false
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -261,6 +266,8 @@ spec:
                           Selector specifies set of labels of the resources that will be synced, if empty
                           then all resources of the given type will be synced.
                         type: object
+                    required:
+                    - enabled
                     type: object
                   persistentVolumeClaims:
                     default:
@@ -268,6 +275,7 @@ spec:
                     description: PersistentVolumeClaims resources sync configuration.
                     properties:
                       enabled:
+                        default: true
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -277,6 +285,8 @@ spec:
                           Selector specifies set of labels of the resources that will be synced, if empty
                           then all resources of the given type will be synced.
                         type: object
+                    required:
+                    - enabled
                     type: object
                   priorityClasses:
                     default:
@@ -284,6 +294,7 @@ spec:
                     description: PriorityClasses resources sync configuration.
                     properties:
                       enabled:
+                        default: false
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -293,6 +304,8 @@ spec:
                           Selector specifies set of labels of the resources that will be synced, if empty
                           then all resources of the given type will be synced.
                         type: object
+                    required:
+                    - enabled
                     type: object
                   secrets:
                     default:
@@ -300,6 +313,7 @@ spec:
                     description: Secrets resources sync configuration.
                     properties:
                       enabled:
+                        default: true
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -316,6 +330,7 @@ spec:
                     description: Services resources sync configuration.
                     properties:
                       enabled:
+                        default: true
                         description: Enabled is an on/off switch for syncing resources.
                         type: boolean
                       selector:
@@ -325,6 +340,8 @@ spec:
                           Selector specifies set of labels of the resources that will be synced, if empty
                           then all resources of the given type will be synced.
                         type: object
+                    required:
+                    - enabled
                     type: object
                 type: object
             type: object

cli/cmds/cluster_create.go

@@ -1,12 +1,14 @@
 package cmds
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"net/url"
 	"os"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -37,6 +39,8 @@ type CreateConfig struct {
 	agentArgs            []string
 	serverEnvs           []string
 	agentEnvs            []string
+	labels               []string
+	annotations          []string
 	persistenceType      string
 	storageClassName     string
 	storageRequestSize   string
@@ -46,6 +50,7 @@ type CreateConfig struct {
 	policy               string
 	mirrorHostNodes      bool
 	customCertsPath      string
+	timeout              time.Duration
 }
 
 func NewClusterCreateCmd(appCtx *AppContext) *cobra.Command {
@@ -110,7 +115,7 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
 			}
 		}
 
-		logrus.Infof("Creating cluster [%s] in namespace [%s]", name, namespace)
+		logrus.Infof("Creating cluster '%s' in namespace '%s'", name, namespace)
 
 		cluster := newCluster(name, namespace, config)
 
@@ -133,19 +138,30 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
 
 		if err := client.Create(ctx, cluster); err != nil {
 			if apierrors.IsAlreadyExists(err) {
-				logrus.Infof("Cluster [%s] already exists", name)
+				logrus.Infof("Cluster '%s' already exists", name)
 			} else {
 				return err
 			}
 		}
 
+		if err := waitForClusterReconciled(ctx, client, cluster, config.timeout); err != nil {
+			return fmt.Errorf("failed to wait for cluster to be reconciled: %w", err)
+		}
+
+		clusterDetails, err := printClusterDetails(cluster)
+		if err != nil {
+			return fmt.Errorf("failed to print cluster details: %w", err)
+		}
+
+		logrus.Info(clusterDetails)
+
 		logrus.Infof("Waiting for cluster to be available..")
 
-		if err := waitForCluster(ctx, client, cluster); err != nil {
+		if err := waitForClusterReady(ctx, client, cluster, config.timeout); err != nil {
 			return fmt.Errorf("failed to wait for cluster to become ready (status: %s): %w", cluster.Status.Phase, err)
 		}
 
-		logrus.Infof("Extracting Kubeconfig for [%s] cluster", name)
+		logrus.Infof("Extracting Kubeconfig for '%s' cluster", name)
 
 		// retry every 5s for at most 2m, or 25 times
 		availableBackoff := wait.Backoff{
@@ -172,8 +188,10 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
 func newCluster(name, namespace string, config *CreateConfig) *v1beta1.Cluster {
 	cluster := &v1beta1.Cluster{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: namespace,
+			Name:        name,
+			Namespace:   namespace,
+			Labels:      parseKeyValuePairs(config.labels, "label"),
+			Annotations: parseKeyValuePairs(config.annotations, "annotation"),
 		},
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Cluster",
@@ -210,7 +228,7 @@ func newCluster(name, namespace string, config *CreateConfig) *v1beta1.Cluster {
 	}
 
 	if config.customCertsPath != "" {
-		cluster.Spec.CustomCAs = v1beta1.CustomCAs{
+		cluster.Spec.CustomCAs = &v1beta1.CustomCAs{
 			Enabled: true,
 			Sources: v1beta1.CredentialSources{
 				ClientCA: v1beta1.CredentialSource{
@@ -256,9 +274,19 @@ func env(envSlice []string) []v1.EnvVar {
 	return envVars
 }
 
-func waitForCluster(ctx context.Context, k8sClient client.Client, cluster *v1beta1.Cluster) error {
+func waitForClusterReconciled(ctx context.Context, k8sClient client.Client, cluster *v1beta1.Cluster, timeout time.Duration) error {
+	return wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, func(ctx context.Context) (bool, error) {
+		key := client.ObjectKeyFromObject(cluster)
+		if err := k8sClient.Get(ctx, key, cluster); err != nil {
+			return false, fmt.Errorf("failed to get resource: %w", err)
+		}
+
+		return cluster.Status.HostVersion != "", nil
+	})
+}
+
+func waitForClusterReady(ctx context.Context, k8sClient client.Client, cluster *v1beta1.Cluster, timeout time.Duration) error {
 	interval := 5 * time.Second
-	timeout := 2 * time.Minute
 
 	return wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) {
 		key := client.ObjectKeyFromObject(cluster)
@@ -341,3 +369,73 @@ func caCertSecret(certName, clusterName, clusterNamespace string, cert, key []by
 		},
 	}
 }
+
+func parseKeyValuePairs(pairs []string, pairType string) map[string]string {
+	resultMap := make(map[string]string)
+
+	for _, p := range pairs {
+		var k, v string
+
+		keyValue := strings.SplitN(p, "=", 2)
+
+		k = keyValue[0]
+		if len(keyValue) == 2 {
+			v = keyValue[1]
+		}
+
+		resultMap[k] = v
+
+		logrus.Debugf("Adding '%s=%s' %s to Cluster", k, v, pairType)
+	}
+
+	return resultMap
+}
+
+const clusterDetailsTemplate = `Cluster details:
+  Mode: {{ .Mode }}
+  Servers: {{ .Servers }}{{ if .Agents }}
+  Agents: {{ .Agents }}{{ end }}
+  Version: {{ if .Version }}{{ .Version }}{{ else }}{{ .HostVersion }}{{ end }} (Host: {{ .HostVersion }})
+  Persistence:
+    Type: {{.Persistence.Type}}{{ if .Persistence.StorageClassName }}
+    StorageClass: {{ .Persistence.StorageClassName }}{{ end }}{{ if .Persistence.StorageRequestSize }}
+    Size: {{ .Persistence.StorageRequestSize }}{{ end }}`
+
+func printClusterDetails(cluster *v1beta1.Cluster) (string, error) {
+	type templateData struct {
+		Mode        v1beta1.ClusterMode
+		Servers     int32
+		Agents      int32
+		Version     string
+		HostVersion string
+		Persistence struct {
+			Type               v1beta1.PersistenceMode
+			StorageClassName   string
+			StorageRequestSize string
+		}
+	}
+
+	data := templateData{
+		Mode:        cluster.Spec.Mode,
+		Servers:     ptr.Deref(cluster.Spec.Servers, 0),
+		Agents:      ptr.Deref(cluster.Spec.Agents, 0),
+		Version:     cluster.Spec.Version,
+		HostVersion: cluster.Status.HostVersion,
+	}
+
+	data.Persistence.Type = cluster.Spec.Persistence.Type
+	data.Persistence.StorageClassName = ptr.Deref(cluster.Spec.Persistence.StorageClassName, "")
+	data.Persistence.StorageRequestSize = cluster.Spec.Persistence.StorageRequestSize
+
+	tmpl, err := template.New("clusterDetails").Parse(clusterDetailsTemplate)
+	if err != nil {
+		return "", err
+	}
+
+	var buf bytes.Buffer
+	if err = tmpl.Execute(&buf, data); err != nil {
+		return "", err
+	}
+
+	return buf.String(), nil
+}

cli/cmds/cluster_create_flags.go

@@ -2,6 +2,7 @@ package cmds
 
 import (
 	"errors"
+	"time"
 
 	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -16,18 +17,21 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) {
 	cmd.Flags().StringVar(&cfg.clusterCIDR, "cluster-cidr", "", "cluster CIDR")
 	cmd.Flags().StringVar(&cfg.serviceCIDR, "service-cidr", "", "service CIDR")
 	cmd.Flags().BoolVar(&cfg.mirrorHostNodes, "mirror-host-nodes", false, "Mirror Host Cluster Nodes")
-	cmd.Flags().StringVar(&cfg.persistenceType, "persistence-type", string(v1beta1.DynamicPersistenceMode), "persistence mode for the nodes (dynamic, ephemeral, static)")
+	cmd.Flags().StringVar(&cfg.persistenceType, "persistence-type", string(v1beta1.DynamicPersistenceMode), "persistence mode for the nodes (dynamic, ephemeral)")
 	cmd.Flags().StringVar(&cfg.storageClassName, "storage-class-name", "", "storage class name for dynamic persistence type")
 	cmd.Flags().StringVar(&cfg.storageRequestSize, "storage-request-size", "", "storage size for dynamic persistence type")
 	cmd.Flags().StringSliceVar(&cfg.serverArgs, "server-args", []string{}, "servers extra arguments")
 	cmd.Flags().StringSliceVar(&cfg.agentArgs, "agent-args", []string{}, "agents extra arguments")
 	cmd.Flags().StringSliceVar(&cfg.serverEnvs, "server-envs", []string{}, "servers extra Envs")
 	cmd.Flags().StringSliceVar(&cfg.agentEnvs, "agent-envs", []string{}, "agents extra Envs")
+	cmd.Flags().StringArrayVar(&cfg.labels, "labels", []string{}, "Labels to add to the cluster object (e.g. key=value)")
+	cmd.Flags().StringArrayVar(&cfg.annotations, "annotations", []string{}, "Annotations to add to the cluster object (e.g. key=value)")
 	cmd.Flags().StringVar(&cfg.version, "version", "", "k3s version")
 	cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual)")
 	cmd.Flags().StringVar(&cfg.kubeconfigServerHost, "kubeconfig-server", "", "override the kubeconfig server host")
 	cmd.Flags().StringVar(&cfg.policy, "policy", "", "The policy to create the cluster in")
 	cmd.Flags().StringVar(&cfg.customCertsPath, "custom-certs", "", "The path for custom certificate directory")
+	cmd.Flags().DurationVar(&cfg.timeout, "timeout", 3*time.Minute, "The timeout for waiting for the cluster to become ready (e.g., 10s, 5m, 1h).")
 }
 
 func validateCreateConfig(cfg *CreateConfig) error {
@@ -40,7 +44,7 @@ func validateCreateConfig(cfg *CreateConfig) error {
 		case v1beta1.EphemeralPersistenceMode, v1beta1.DynamicPersistenceMode:
 			return nil
 		default:
-			return errors.New(`persistence-type should be one of "dynamic", "ephemeral" or "static"`)
+			return errors.New(`persistence-type should be one of "dynamic" or "ephemeral"`)
 		}
 	}
 

cli/cmds/cluster_create_test.go

@@ -0,0 +1,95 @@
+package cmds
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/utils/ptr"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
+)
+
+func Test_printClusterDetails(t *testing.T) {
+	tests := []struct {
+		name    string
+		cluster *v1beta1.Cluster
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "simple cluster",
+			cluster: &v1beta1.Cluster{
+				Spec: v1beta1.ClusterSpec{
+					Mode:    v1beta1.SharedClusterMode,
+					Version: "123",
+					Persistence: v1beta1.PersistenceConfig{
+						Type: v1beta1.DynamicPersistenceMode,
+					},
+				},
+				Status: v1beta1.ClusterStatus{
+					HostVersion: "456",
+				},
+			},
+			want: `Cluster details:
+  Mode: shared
+  Servers: 0
+  Version: 123 (Host: 456)
+  Persistence:
+    Type: dynamic`,
+		},
+		{
+			name: "simple cluster with no version",
+			cluster: &v1beta1.Cluster{
+				Spec: v1beta1.ClusterSpec{
+					Mode: v1beta1.SharedClusterMode,
+					Persistence: v1beta1.PersistenceConfig{
+						Type: v1beta1.DynamicPersistenceMode,
+					},
+				},
+				Status: v1beta1.ClusterStatus{
+					HostVersion: "456",
+				},
+			},
+			want: `Cluster details:
+  Mode: shared
+  Servers: 0
+  Version: 456 (Host: 456)
+  Persistence:
+    Type: dynamic`,
+		},
+		{
+			name: "cluster with agents",
+			cluster: &v1beta1.Cluster{
+				Spec: v1beta1.ClusterSpec{
+					Mode:   v1beta1.SharedClusterMode,
+					Agents: ptr.To[int32](3),
+					Persistence: v1beta1.PersistenceConfig{
+						Type:               v1beta1.DynamicPersistenceMode,
+						StorageClassName:   ptr.To("local-path"),
+						StorageRequestSize: "3gb",
+					},
+				},
+				Status: v1beta1.ClusterStatus{
+					HostVersion: "456",
+				},
+			},
+			want: `Cluster details:
+  Mode: shared
+  Servers: 0
+  Agents: 3
+  Version: 456 (Host: 456)
+  Persistence:
+    Type: dynamic
+    StorageClass: local-path
+    Size: 3gb`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			clusterDetails, err := printClusterDetails(tt.cluster)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.want, clusterDetails)
+		})
+	}
+}

cli/cmds/cluster_delete.go

@@ -48,7 +48,7 @@ func delete(appCtx *AppContext) func(cmd *cobra.Command, args []string) error {
 
 		namespace := appCtx.Namespace(name)
 
-		logrus.Infof("Deleting [%s] cluster in namespace [%s]", name, namespace)
+		logrus.Infof("Deleting '%s' cluster in namespace '%s'", name, namespace)
 
 		cluster := v1beta1.Cluster{
 			ObjectMeta: metav1.ObjectMeta{

cli/cmds/policy_create.go

@@ -18,7 +18,11 @@ import (
 )
 
 type VirtualClusterPolicyCreateConfig struct {
-	mode string
+	mode        string
+	labels      []string
+	annotations []string
+	namespaces  []string
+	overwrite   bool
 }
 
 func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command {
@@ -41,6 +45,10 @@ func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command {
 	}
 
 	cmd.Flags().StringVar(&config.mode, "mode", "shared", "The allowed mode type of the policy")
+	cmd.Flags().StringArrayVar(&config.labels, "labels", []string{}, "Labels to add to the policy object (e.g. key=value)")
+	cmd.Flags().StringArrayVar(&config.annotations, "annotations", []string{}, "Annotations to add to the policy object (e.g. key=value)")
+	cmd.Flags().StringSliceVar(&config.namespaces, "namespace", []string{}, "The namespaces where to bind the policy")
+	cmd.Flags().BoolVar(&config.overwrite, "overwrite", false, "Overwrite namespace binding of existing policy")
 
 	return cmd
 }
@@ -51,9 +59,12 @@ func policyCreateAction(appCtx *AppContext, config *VirtualClusterPolicyCreateCo
 		client := appCtx.Client
 		policyName := args[0]
 
-		_, err := createPolicy(ctx, client, v1beta1.ClusterMode(config.mode), policyName)
+		_, err := createPolicy(ctx, client, config, policyName)
+		if err != nil {
+			return err
+		}
 
-		return err
+		return bindPolicyToNamespaces(ctx, client, config, policyName)
 	}
 }
 
@@ -71,7 +82,7 @@ func createNamespace(ctx context.Context, client client.Client, name, policyName
 			return err
 		}
 
-		logrus.Infof(`Creating namespace [%s]`, name)
+		logrus.Infof(`Creating namespace '%s'`, name)
 
 		if err := client.Create(ctx, ns); err != nil {
 			return err
@@ -81,19 +92,21 @@ func createNamespace(ctx context.Context, client client.Client, name, policyName
 	return nil
 }
 
-func createPolicy(ctx context.Context, client client.Client, mode v1beta1.ClusterMode, policyName string) (*v1beta1.VirtualClusterPolicy, error) {
-	logrus.Infof("Creating policy [%s]", policyName)
+func createPolicy(ctx context.Context, client client.Client, config *VirtualClusterPolicyCreateConfig, policyName string) (*v1beta1.VirtualClusterPolicy, error) {
+	logrus.Infof("Creating policy '%s'", policyName)
 
 	policy := &v1beta1.VirtualClusterPolicy{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: policyName,
+			Name:        policyName,
+			Labels:      parseKeyValuePairs(config.labels, "label"),
+			Annotations: parseKeyValuePairs(config.annotations, "annotation"),
 		},
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "VirtualClusterPolicy",
 			APIVersion: "k3k.io/v1beta1",
 		},
 		Spec: v1beta1.VirtualClusterPolicySpec{
-			AllowedMode: mode,
+			AllowedMode: v1beta1.ClusterMode(config.mode),
 		},
 	}
 
@@ -102,8 +115,67 @@ func createPolicy(ctx context.Context, client client.Client, mode v1beta1.Cluste
 			return nil, err
 		}
 
-		logrus.Infof("Policy [%s] already exists", policyName)
+		logrus.Infof("Policy '%s' already exists", policyName)
 	}
 
 	return policy, nil
 }
+
+func bindPolicyToNamespaces(ctx context.Context, client client.Client, config *VirtualClusterPolicyCreateConfig, policyName string) error {
+	var errs []error
+
+	for _, namespace := range config.namespaces {
+		var ns v1.Namespace
+		if err := client.Get(ctx, types.NamespacedName{Name: namespace}, &ns); err != nil {
+			if apierrors.IsNotFound(err) {
+				logrus.Warnf(`Namespace '%s' not found, skipping`, namespace)
+			} else {
+				errs = append(errs, err)
+			}
+
+			continue
+		}
+
+		if ns.Labels == nil {
+			ns.Labels = map[string]string{}
+		}
+
+		oldPolicy := ns.Labels[policy.PolicyNameLabelKey]
+
+		// same policy found, no need to update
+		if oldPolicy == policyName {
+			logrus.Debugf(`Policy '%s' already bound to namespace '%s'`, policyName, namespace)
+			continue
+		}
+
+		// no old policy, safe to update
+		if oldPolicy == "" {
+			if err := client.Update(ctx, &ns); err != nil {
+				errs = append(errs, err)
+			} else {
+				logrus.Infof(`Added policy '%s' to namespace '%s'`, policyName, namespace)
+			}
+
+			continue
+		}
+
+		// different policy, warn or check for overwrite flag
+		if oldPolicy != policyName {
+			if config.overwrite {
+				logrus.Infof(`Found policy '%s' bound to namespace '%s'. Overwriting it with '%s'`, oldPolicy, namespace, policyName)
+
+				ns.Labels[policy.PolicyNameLabelKey] = policyName
+
+				if err := client.Update(ctx, &ns); err != nil {
+					errs = append(errs, err)
+				} else {
+					logrus.Infof(`Added policy '%s' to namespace '%s'`, policyName, namespace)
+				}
+			} else {
+				logrus.Warnf(`Found policy '%s' bound to namespace '%s'. Skipping. To overwrite it use the --overwrite flag`, oldPolicy, namespace)
+			}
+		}
+	}
+
+	return errors.Join(errs...)
+}

cli/cmds/policy_delete.go

@@ -31,13 +31,17 @@ func policyDeleteAction(appCtx *AppContext) func(cmd *cobra.Command, args []stri
 		policy.Name = name
 
 		if err := client.Delete(ctx, policy); err != nil {
-			if apierrors.IsNotFound(err) {
-				logrus.Warnf("Policy not found")
-			} else {
+			if !apierrors.IsNotFound(err) {
 				return err
 			}
+
+			logrus.Warnf("Policy '%s' not found", name)
+
+			return nil
 		}
 
+		logrus.Infof("Policy '%s' deleted", name)
+
 		return nil
 	}
 }

cli/cmds/root.go

@@ -34,9 +34,10 @@ func NewRootCmd() *cobra.Command {
 	appCtx := &AppContext{}
 
 	rootCmd := &cobra.Command{
-		Use:     "k3kcli",
-		Short:   "CLI for K3K",
-		Version: buildinfo.Version,
+		SilenceUsage: true,
+		Use:          "k3kcli",
+		Short:        "CLI for K3K",
+		Version:      buildinfo.Version,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			InitializeConfig(cmd)
 

docs/cli/k3kcli_cluster_create.md

@@ -18,14 +18,16 @@ k3kcli cluster create [command options] NAME
       --agent-args strings            agents extra arguments
       --agent-envs strings            agents extra Envs
       --agents int                    number of agents
+      --annotations stringArray       Annotations to add to the cluster object (e.g. key=value)
       --cluster-cidr string           cluster CIDR
       --custom-certs string           The path for custom certificate directory
   -h, --help                          help for create
       --kubeconfig-server string      override the kubeconfig server host
+      --labels stringArray            Labels to add to the cluster object (e.g. key=value)
       --mirror-host-nodes             Mirror Host Cluster Nodes
       --mode string                   k3k mode type (shared, virtual) (default "shared")
   -n, --namespace string              namespace of the k3k cluster
-      --persistence-type string       persistence mode for the nodes (dynamic, ephemeral, static) (default "dynamic")
+      --persistence-type string       persistence mode for the nodes (dynamic, ephemeral) (default "dynamic")
       --policy string                 The policy to create the cluster in
       --server-args strings           servers extra arguments
       --server-envs strings           servers extra Envs
@@ -33,6 +35,7 @@ k3kcli cluster create [command options] NAME
       --service-cidr string           service CIDR
       --storage-class-name string     storage class name for dynamic persistence type
       --storage-request-size string   storage size for dynamic persistence type
+      --timeout duration              The timeout for waiting for the cluster to become ready (e.g., 10s, 5m, 1h). (default 3m0s)
       --token string                  token of the cluster
       --version string                k3s version
 ```

docs/cli/k3kcli_policy_create.md

@@ -15,8 +15,12 @@ k3kcli policy create [command options] NAME
 ### Options
 
 ```
-  -h, --help          help for create
-      --mode string   The allowed mode type of the policy (default "shared")
+      --annotations stringArray   Annotations to add to the policy object (e.g. key=value)
+  -h, --help                      help for create
+      --labels stringArray        Labels to add to the policy object (e.g. key=value)
+      --mode string               The allowed mode type of the policy (default "shared")
+      --namespace strings         The namespaces where to bind the policy
+      --overwrite                 Overwrite namespace binding of existing policy
 ```
 
 ### Options inherited from parent commands

docs/crds/crd-docs.md

@@ -152,7 +152,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. |  |  |
+| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. | true |  |
 | `selector` _object (keys:string, values:string)_ | Selector specifies set of labels of the resources that will be synced, if empty<br />then all resources of the given type will be synced. |  |  |
 
 
@@ -208,7 +208,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `enabled` _boolean_ | Enabled toggles this feature on or off. |  |  |
+| `enabled` _boolean_ | Enabled toggles this feature on or off. | true |  |
 | `sources` _[CredentialSources](#credentialsources)_ | Sources defines the sources for all required custom CA certificates. |  |  |
 
 
@@ -226,7 +226,7 @@ _Appears in:_
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `ingress` _[IngressConfig](#ingressconfig)_ | Ingress specifies options for exposing the API server through an Ingress. |  |  |
-| `loadbalancer` _[LoadBalancerConfig](#loadbalancerconfig)_ | LoadBalancer specifies options for exposing the API server through a LoadBalancer service. |  |  |
+| `loadBalancer` _[LoadBalancerConfig](#loadbalancerconfig)_ | LoadBalancer specifies options for exposing the API server through a LoadBalancer service. |  |  |
 | `nodePort` _[NodePortConfig](#nodeportconfig)_ | NodePort specifies options for exposing the API server through NodePort. |  |  |
 
 
@@ -260,7 +260,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. |  |  |
+| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. | false |  |
 | `selector` _object (keys:string, values:string)_ | Selector specifies set of labels of the resources that will be synced, if empty<br />then all resources of the given type will be synced. |  |  |
 
 
@@ -342,7 +342,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. |  |  |
+| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. | true |  |
 | `selector` _object (keys:string, values:string)_ | Selector specifies set of labels of the resources that will be synced, if empty<br />then all resources of the given type will be synced. |  |  |
 
 
@@ -373,7 +373,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. |  |  |
+| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. | false |  |
 | `selector` _object (keys:string, values:string)_ | Selector specifies set of labels of the resources that will be synced, if empty<br />then all resources of the given type will be synced. |  |  |
 
 
@@ -390,7 +390,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. |  |  |
+| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. | true |  |
 | `selector` _object (keys:string, values:string)_ | Selector specifies set of labels of the resources that will be synced, if empty<br />then all resources of the given type will be synced. |  |  |
 
 
@@ -407,7 +407,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. |  |  |
+| `enabled` _boolean_ | Enabled is an on/off switch for syncing resources. | true |  |
 | `selector` _object (keys:string, values:string)_ | Selector specifies set of labels of the resources that will be synced, if empty<br />then all resources of the given type will be synced. |  |  |
 
 
@@ -426,7 +426,7 @@ _Appears in:_
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `services` _[ServiceSyncConfig](#servicesyncconfig)_ | Services resources sync configuration. | \{ enabled:true \} |  |
-| `configmaps` _[ConfigMapSyncConfig](#configmapsyncconfig)_ | ConfigMaps resources sync configuration. | \{ enabled:true \} |  |
+| `configMaps` _[ConfigMapSyncConfig](#configmapsyncconfig)_ | ConfigMaps resources sync configuration. | \{ enabled:true \} |  |
 | `secrets` _[SecretSyncConfig](#secretsyncconfig)_ | Secrets resources sync configuration. | \{ enabled:true \} |  |
 | `ingresses` _[IngressSyncConfig](#ingresssyncconfig)_ | Ingresses resources sync configuration. | \{ enabled:false \} |  |
 | `persistentVolumeClaims` _[PersistentVolumeClaimSyncConfig](#persistentvolumeclaimsyncconfig)_ | PersistentVolumeClaims resources sync configuration. | \{ enabled:true \} |  |

examples/multiple-servers.yaml

@@ -1,19 +0,0 @@
-apiVersion: k3k.io/v1beta1
-kind: Cluster
-metadata:
-  name: example1
-spec:
-  mode: "shared"
-  servers: 1
-  agents: 3
-  token: test
-  version: v1.26.0-k3s2
-  clusterCIDR: 10.30.0.0/16
-  serviceCIDR: 10.31.0.0/16
-  clusterDNS: 10.30.0.10
-  serverArgs:
-  - "--write-kubeconfig-mode=777"
-  expose:
-    ingress:
-      enabled: true
-      ingressClassName: "nginx"

examples/shared-multiple-servers.yaml

@@ -0,0 +1,15 @@
+apiVersion: k3k.io/v1beta1
+kind: Cluster
+metadata:
+  name: shared-multiple-servers
+spec:
+  mode: shared
+  servers: 3
+  agents: 3
+  version: v1.33.1-k3s1
+  serverArgs:
+  - "--write-kubeconfig-mode=777"
+  tlsSANs:
+  - myserver.app
+  expose:
+    nodePort: {}

examples/shared-single-server.yaml

@@ -0,0 +1,14 @@
+apiVersion: k3k.io/v1beta1
+kind: Cluster
+metadata:
+  name: shared-single-server
+spec:
+  mode: shared
+  servers: 1
+  version: v1.33.1-k3s1
+  serverArgs:
+  - "--write-kubeconfig-mode=777"
+  tlsSANs:
+  - myserver.app
+  expose:
+    nodePort: {}

examples/single-server.yaml

@@ -1,19 +0,0 @@
-apiVersion: k3k.io/v1beta1
-kind: Cluster
-metadata:
-  name: single-server
-spec:
-  mode: "shared"
-  servers: 1
-  agents: 3
-  token: test
-  version: v1.26.0-k3s2
-  clusterCIDR: 10.30.0.0/16
-  serviceCIDR: 10.31.0.0/16
-  clusterDNS: 10.30.0.10
-  serverArgs:
-  - "--write-kubeconfig-mode=777"
-  expose:
-    ingress:
-      enabled: true
-      ingressClassName: "nginx"

examples/virtual-server.yaml

@@ -0,0 +1,13 @@
+apiVersion: k3k.io/v1beta1
+kind: Cluster
+metadata:
+  name: virtual-server
+spec:
+  mode: virtual
+  servers: 3
+  agents: 3
+  version: v1.33.1-k3s1
+  tlsSANs:
+  - myserver.app
+  expose:
+    nodePort: {}

examples/virtualclusterpolicy.yaml

@@ -2,8 +2,8 @@ apiVersion: k3k.io/v1beta1
 kind: VirtualClusterPolicy
 metadata:
   name: policy-example
-# spec:
-  # disableNetworkPolicy: false
-  # allowedMode: "shared"
+spec:
+  allowedMode: shared
+  disableNetworkPolicy: true
   # podSecurityAdmissionLevel: "baseline"
   # defaultPriorityClass: "lowpriority"

go.mod

@@ -1,6 +1,6 @@
 module github.com/rancher/k3k
 
-go 1.24.2
+go 1.24.10
 
 replace (
 	github.com/google/cel-go => github.com/google/cel-go v0.20.1
@@ -18,8 +18,10 @@ require (
 	github.com/onsi/gomega v1.36.0
 	github.com/rancher/dynamiclistener v1.27.5
 	github.com/sirupsen/logrus v1.9.3
-	github.com/spf13/viper v1.20.1
-	github.com/stretchr/testify v1.10.0
+	github.com/spf13/cobra v1.10.1
+	github.com/spf13/pflag v1.0.10
+	github.com/spf13/viper v1.21.0
+	github.com/stretchr/testify v1.11.1
 	github.com/testcontainers/testcontainers-go v0.35.0
 	github.com/testcontainers/testcontainers-go/modules/k3s v0.35.0
 	github.com/virtual-kubelet/virtual-kubelet v1.11.1-0.20250530103808-c9f64e872803
@@ -28,17 +30,17 @@ require (
 	go.uber.org/zap v1.27.0
 	gopkg.in/yaml.v2 v2.4.0
 	helm.sh/helm/v3 v3.14.4
-	k8s.io/api v0.31.4
-	k8s.io/apiextensions-apiserver v0.31.4
-	k8s.io/apimachinery v0.31.4
-	k8s.io/apiserver v0.31.4
-	k8s.io/cli-runtime v0.31.4
-	k8s.io/client-go v0.31.4
-	k8s.io/component-base v0.31.4
-	k8s.io/component-helpers v0.31.4
-	k8s.io/kubectl v0.31.4
-	k8s.io/kubelet v0.31.4
-	k8s.io/kubernetes v1.31.4
+	k8s.io/api v0.31.13
+	k8s.io/apiextensions-apiserver v0.31.13
+	k8s.io/apimachinery v0.31.13
+	k8s.io/apiserver v0.31.13
+	k8s.io/cli-runtime v0.31.13
+	k8s.io/client-go v0.31.13
+	k8s.io/component-base v0.31.13
+	k8s.io/component-helpers v0.31.13
+	k8s.io/kubectl v0.31.13
+	k8s.io/kubelet v0.31.13
+	k8s.io/kubernetes v1.31.13
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
 	sigs.k8s.io/controller-runtime v0.19.4
 )
@@ -86,7 +88,7 @@ require (
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
@@ -96,7 +98,7 @@ require (
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -153,7 +155,7 @@ require (
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -164,15 +166,13 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rubenv/sql-migrate v1.7.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/sagikazarmark/locafero v0.11.0 // indirect
 	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
-	github.com/sourcegraph/conc v0.3.0 // indirect
-	github.com/spf13/afero v1.12.0 // indirect
-	github.com/spf13/cast v1.7.1 // indirect
-	github.com/spf13/cobra v1.9.1
-	github.com/spf13/pflag v1.0.6
+	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
+	github.com/spf13/afero v1.15.0 // indirect
+	github.com/spf13/cast v1.10.0 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
@@ -196,16 +196,17 @@ require (
 	go.opentelemetry.io/otel/trace v1.33.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.38.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.40.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
-	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/net v0.42.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
-	golang.org/x/sync v0.14.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/term v0.32.0 // indirect
-	golang.org/x/text v0.25.0 // indirect
+	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sys v0.34.0 // indirect
+	golang.org/x/term v0.33.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
 	golang.org/x/time v0.9.0 // indirect
-	golang.org/x/tools v0.26.0 // indirect
+	golang.org/x/tools v0.35.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 // indirect
@@ -216,7 +217,7 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.130.1
-	k8s.io/kms v0.31.4 // indirect
+	k8s.io/kms v0.31.13 // indirect
 	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
 	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect

go.sum

@@ -137,8 +137,8 @@ github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6
 github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
-github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
-github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@@ -166,8 +166,8 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
-github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
-github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
+github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -353,8 +353,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
-github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
-github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
+github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
+github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
@@ -388,8 +388,8 @@ github.com/rubenv/sql-migrate v1.7.1 h1:f/o0WgfO/GqNuVg+6801K/KW3WdDSupzSjDYODmi
 github.com/rubenv/sql-migrate v1.7.1/go.mod h1:Ob2Psprc0/3ggbM6wCzyYVFFuc6FyZrb2AS+ezLDFb4=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
-github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
+github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
+github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
 github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4=
@@ -404,18 +404,19 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
 github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
-github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
-github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
-github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
-github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
-github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
-github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
-github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
-github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
-github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
-github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
-github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
+github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
+github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
+github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
+github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
+github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
+github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
+github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
+github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
+github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
 github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
 github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -433,8 +434,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/testcontainers/testcontainers-go v0.35.0 h1:uADsZpTKFAtp8SLK+hMwSaa+X+JiERHtd4sQAFmXeMo=
@@ -520,6 +522,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -527,8 +531,9 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
 golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
+golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
+golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
@@ -560,8 +565,9 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
 golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
+golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
+golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
@@ -578,8 +584,9 @@ golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
 golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -603,8 +610,9 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
 golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
+golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
@@ -617,8 +625,9 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
 golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
 golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
 golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -633,8 +642,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
-golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
+golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0=
+golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -647,8 +656,8 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk=
-google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc=
+google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 h1:1hfbdAfFbkmpg41000wDVqr7jUpK/Yo+LPnIxxGzmkg=
+google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3/go.mod h1:5RBcpGRxr25RbDzY5w+dmaqpSEvl8Gwl1x2CICf60ic=
 google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q=
 google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 h1:TqExAhdPaB60Ux47Cn0oLV07rGnxZzIsaRhQaqS666A=
@@ -700,34 +709,34 @@ helm.sh/helm/v3 v3.14.4 h1:6FSpEfqyDalHq3kUr4gOMThhgY55kXUEjdQoyODYnrM=
 helm.sh/helm/v3 v3.14.4/go.mod h1:Tje7LL4gprZpuBNTbG34d1Xn5NmRT3OWfBRwpOSer9I=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.31.4 h1:I2QNzitPVsPeLQvexMEsj945QumYraqv9m74isPDKhM=
-k8s.io/api v0.31.4/go.mod h1:d+7vgXLvmcdT1BCo79VEgJxHHryww3V5np2OYTr6jdw=
-k8s.io/apiextensions-apiserver v0.31.4 h1:FxbqzSvy92Ca9DIs5jqot883G0Ln/PGXfm/07t39LS0=
-k8s.io/apiextensions-apiserver v0.31.4/go.mod h1:hIW9YU8UsqZqIWGG99/gsdIU0Ar45Qd3A12QOe/rvpg=
-k8s.io/apimachinery v0.31.4 h1:8xjE2C4CzhYVm9DGf60yohpNUh5AEBnPxCryPBECmlM=
-k8s.io/apimachinery v0.31.4/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
-k8s.io/apiserver v0.31.4 h1:JbtnTaXVYEAYIHJil6Wd74Wif9sd8jVcBw84kwEmp7o=
-k8s.io/apiserver v0.31.4/go.mod h1:JJjoTjZ9PTMLdIFq7mmcJy2B9xLN3HeAUebW6xZyIP0=
-k8s.io/cli-runtime v0.31.4 h1:iczCWiyXaotW+hyF5cWP8RnEYBCzZfJUF6otJ2m9mw0=
-k8s.io/cli-runtime v0.31.4/go.mod h1:0/pRzAH7qc0hWx40ut1R4jLqiy2w/KnbqdaAI2eFG8U=
-k8s.io/client-go v0.31.4 h1:t4QEXt4jgHIkKKlx06+W3+1JOwAFU/2OPiOo7H92eRQ=
-k8s.io/client-go v0.31.4/go.mod h1:kvuMro4sFYIa8sulL5Gi5GFqUPvfH2O/dXuKstbaaeg=
-k8s.io/component-base v0.31.4 h1:wCquJh4ul9O8nNBSB8N/o8+gbfu3BVQkVw9jAUY/Qtw=
-k8s.io/component-base v0.31.4/go.mod h1:G4dgtf5BccwiDT9DdejK0qM6zTK0jwDGEKnCmb9+u/s=
-k8s.io/component-helpers v0.31.4 h1:pqokuXozyWVrVBMmx0AMcKqNWqXhR00OZvpAE5hG5NM=
-k8s.io/component-helpers v0.31.4/go.mod h1:Ddq5GYRK/1uNoPNgJh9N5osPutvBweQEcIG6b8kcvgQ=
+k8s.io/api v0.31.13 h1:sco9Cq2pY4Ysv9qZiWzcR97MmA/35nwYQ/VCTzOcWmc=
+k8s.io/api v0.31.13/go.mod h1:4D8Ry8RqqLDemNLwGYC6v5wOy51N7hitr4WQ6oSWfLY=
+k8s.io/apiextensions-apiserver v0.31.13 h1:8xtWKVpV/YbYX0UX2k6w+cgxfxKhX0UNGuo/VXAdg8g=
+k8s.io/apiextensions-apiserver v0.31.13/go.mod h1:zxpMLWXBxnJqKUIruJ+ulP+Xlfe5lPZPxq1z0cLwA2U=
+k8s.io/apimachinery v0.31.13 h1:rkG0EiBkBkEzURo/8dKGx/oBF202Z2LqHuSD8Cm3bG4=
+k8s.io/apimachinery v0.31.13/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/apiserver v0.31.13 h1:Ke9/X2m3vHSgsminpAbUxULDNMbvAfjrRX73Gqx6CZc=
+k8s.io/apiserver v0.31.13/go.mod h1:5nBPhL2g7am/CS+/OI5A6+olEbo0C7tQ8QNDODLd+WY=
+k8s.io/cli-runtime v0.31.13 h1:oz37PuIe4JyUDfTf8JKcZye1obyYAwF146gRpcj+AR8=
+k8s.io/cli-runtime v0.31.13/go.mod h1:x6QI7U97fvrplKgd3JEvCpoZKR9AorjvDjBEr1GZG+g=
+k8s.io/client-go v0.31.13 h1:Q0LG51uFbzNd9fzIj5ilA0Sm1wUholHvDaNwVKzqdCA=
+k8s.io/client-go v0.31.13/go.mod h1:UB4yTzQeRAv+vULOKp2jdqA5LSwV55bvc3RQ5tM48LM=
+k8s.io/component-base v0.31.13 h1:/uVLq7yHk9azReqeCFAZSr/8NXydzpz7yDZ6p/yiwBQ=
+k8s.io/component-base v0.31.13/go.mod h1:uMXtKNyDqeNdZYL6SRCr9wB6FutL9pOlQmkK2dRVAKQ=
+k8s.io/component-helpers v0.31.13 h1:Yy7j+Va7u6v0DXaKqMEOfIcq5pFnvzFcSGM58/lskeA=
+k8s.io/component-helpers v0.31.13/go.mod h1:nXTLwkwCjXcrPG62D0IYiKuKi6JkFM2mBe2myrOUeug=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
-k8s.io/kms v0.31.4 h1:DVk9T1PHxG7IUMfWs1sDhBTbzGnM7lhMJO8lOzOzTIs=
-k8s.io/kms v0.31.4/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94=
+k8s.io/kms v0.31.13 h1:pJCG79BqdCmGetUsETwKfq+OE/D3M1DdqH14EKQl0lI=
+k8s.io/kms v0.31.13/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94=
 k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y=
 k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4=
-k8s.io/kubectl v0.31.4 h1:c8Af8xd1VjyoKyWMW0xHv2+tYxEjne8s6OOziMmaD10=
-k8s.io/kubectl v0.31.4/go.mod h1:0E0rpXg40Q57wRE6LB9su+4tmwx1IzZrmIEvhQPk0i4=
-k8s.io/kubelet v0.31.4 h1:6TokbMv+HnFG7Oe9tVS/J0VPGdC4GnsQZXuZoo7Ixi8=
-k8s.io/kubelet v0.31.4/go.mod h1:8ZM5LZyANoVxUtmayUxD/nsl+6GjREo7kSanv8AoL4U=
-k8s.io/kubernetes v1.31.4 h1:VQDX52gTQnq8C/jCo48AQuDsWbWMh9XXxhQRDYjgakw=
-k8s.io/kubernetes v1.31.4/go.mod h1:9xmT2buyTYj8TRKwRae7FcuY8k5+xlxv7VivvO0KKfs=
+k8s.io/kubectl v0.31.13 h1:VcSyzFsZ7Fi991FzK80hy+9clUIhChbnQg2L6eZRQzA=
+k8s.io/kubectl v0.31.13/go.mod h1:IxUKvsKrvqEL7NcaBCQCVDLzcYghu8b9yMiYKx8nYho=
+k8s.io/kubelet v0.31.13 h1:wN9NXmj9DRFTMph1EhAtdQ6+UfEHKV3B7XMKcJr122c=
+k8s.io/kubelet v0.31.13/go.mod h1:DxEqJViO7GE5dZXvEJGsP5HORNTSj9MhMQi1JDirCQs=
+k8s.io/kubernetes v1.31.13 h1:c/YugS3TqG6YQMNRclrcWVabgIuqyap++lM5AuCtD5M=
+k8s.io/kubernetes v1.31.13/go.mod h1:9xmT2buyTYj8TRKwRae7FcuY8k5+xlxv7VivvO0KKfs=
 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro=
 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=

k3k-kubelet/controller/syncer/configmap.go

@@ -100,6 +100,10 @@ func (c *ConfigMapSyncer) Reconcile(ctx context.Context, req reconcile.Request)
 
 	syncedConfigMap := c.translateConfigMap(&virtualConfigMap)
 
+	if err := controllerutil.SetControllerReference(&cluster, syncedConfigMap, c.HostClient.Scheme()); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	// handle deletion
 	if !virtualConfigMap.DeletionTimestamp.IsZero() {
 		// deleting the synced configMap if exist

k3k-kubelet/controller/syncer/configmap_test.go

@@ -40,13 +40,6 @@ var ConfigMapTests = func() {
 				GenerateName: "cluster-",
 				Namespace:    namespace,
 			},
-			Spec: v1beta1.ClusterSpec{
-				Sync: &v1beta1.SyncConfig{
-					ConfigMaps: v1beta1.ConfigMapSyncConfig{
-						Enabled: true,
-					},
-				},
-			},
 		}
 		err = hostTestEnv.k8sClient.Create(ctx, &cluster)
 		Expect(err).NotTo(HaveOccurred())

k3k-kubelet/controller/syncer/ingress.go

@@ -97,6 +97,7 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	}
 
 	syncedIngress := r.ingress(&virtIngress)
+
 	if err := controllerutil.SetControllerReference(&cluster, syncedIngress, r.HostClient.Scheme()); err != nil {
 		return reconcile.Result{}, err
 	}

k3k-kubelet/controller/syncer/persistentvolumeclaims_test.go

@@ -41,13 +41,6 @@ var PVCTests = func() {
 				GenerateName: "cluster-",
 				Namespace:    namespace,
 			},
-			Spec: v1beta1.ClusterSpec{
-				Sync: &v1beta1.SyncConfig{
-					PersistentVolumeClaims: v1beta1.PersistentVolumeClaimSyncConfig{
-						Enabled: true,
-					},
-				},
-			},
 		}
 		err = hostTestEnv.k8sClient.Create(ctx, &cluster)
 		Expect(err).NotTo(HaveOccurred())

k3k-kubelet/controller/syncer/pod.go

@@ -5,6 +5,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/component-helpers/storage/volume"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -113,9 +114,14 @@ func (r *PodReconciler) reconcilePodWithPVC(ctx context.Context, pod *v1.Pod, pv
 		return ctrlruntimeclient.IgnoreNotFound(err)
 	}
 
+	pv := r.pseudoPV(&pvc)
+
+	if pod.DeletionTimestamp != nil {
+		return r.handlePodDeletion(ctx, pv)
+	}
+
 	log.Info("Creating pseudo Persistent Volume")
 
-	pv := r.pseudoPV(&pvc)
 	if err := r.VirtualClient.Create(ctx, pv); err != nil {
 		return ctrlruntimeclient.IgnoreAlreadyExists(err)
 	}
@@ -188,3 +194,22 @@ func (r *PodReconciler) pseudoPV(obj *v1.PersistentVolumeClaim) *v1.PersistentVo
 		},
 	}
 }
+
+func (r *PodReconciler) handlePodDeletion(ctx context.Context, pv *v1.PersistentVolume) error {
+	var currentPV v1.PersistentVolume
+	if err := r.VirtualClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(pv), &currentPV); err != nil {
+		return ctrlruntimeclient.IgnoreNotFound(err)
+	}
+
+	pvPatch := currentPV.DeepCopy()
+	pvPatch.Spec.ClaimRef = nil
+	pvPatch.Status.Phase = v1.VolumeReleased
+
+	controllerutil.RemoveFinalizer(pvPatch, "kubernetes.io/pv-protection")
+
+	if err := r.VirtualClient.Status().Update(ctx, pvPatch); err != nil {
+		return err
+	}
+
+	return ctrlruntimeclient.IgnoreNotFound(r.VirtualClient.Delete(ctx, &currentPV))
+}

k3k-kubelet/controller/syncer/priorityclass.go

@@ -117,6 +117,10 @@ func (r *PriorityClassSyncer) Reconcile(ctx context.Context, req reconcile.Reque
 
 	hostPriorityClass := r.translatePriorityClass(priorityClass)
 
+	if err := controllerutil.SetControllerReference(&cluster, hostPriorityClass, r.HostClient.Scheme()); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	// handle deletion
 	if !priorityClass.DeletionTimestamp.IsZero() {
 		// deleting the synced service if exists

k3k-kubelet/controller/syncer/secret.go

@@ -100,6 +100,10 @@ func (s *SecretSyncer) Reconcile(ctx context.Context, req reconcile.Request) (re
 
 	syncedSecret := s.translateSecret(&virtualSecret)
 
+	if err := controllerutil.SetControllerReference(&cluster, syncedSecret, s.HostClient.Scheme()); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	// handle deletion
 	if !virtualSecret.DeletionTimestamp.IsZero() {
 		// deleting the synced secret if exist

k3k-kubelet/controller/syncer/secret_test.go

@@ -40,13 +40,6 @@ var SecretTests = func() {
 				GenerateName: "cluster-",
 				Namespace:    namespace,
 			},
-			Spec: v1beta1.ClusterSpec{
-				Sync: &v1beta1.SyncConfig{
-					Secrets: v1beta1.SecretSyncConfig{
-						Enabled: true,
-					},
-				},
-			},
 		}
 		err = hostTestEnv.k8sClient.Create(ctx, &cluster)
 		Expect(err).NotTo(HaveOccurred())

k3k-kubelet/controller/syncer/service.go

@@ -75,6 +75,7 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	}
 
 	syncedService := r.service(&virtService)
+
 	if err := controllerutil.SetControllerReference(&cluster, syncedService, r.HostClient.Scheme()); err != nil {
 		return reconcile.Result{}, err
 	}

k3k-kubelet/controller/syncer/service_test.go

@@ -41,13 +41,6 @@ var ServiceTests = func() {
 				GenerateName: "cluster-",
 				Namespace:    namespace,
 			},
-			Spec: v1beta1.ClusterSpec{
-				Sync: &v1beta1.SyncConfig{
-					Services: v1beta1.ServiceSyncConfig{
-						Enabled: true,
-					},
-				},
-			},
 		}
 		err = hostTestEnv.k8sClient.Create(ctx, &cluster)
 		Expect(err).NotTo(HaveOccurred())

k3k-kubelet/controller/syncer/syncer_suite_test.go

@@ -92,7 +92,7 @@ func NewTestEnv() *TestEnv {
 	By("bootstrapping test environment")
 
 	testEnv := &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")},
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")},
 		ErrorIfCRDPathMissing: true,
 		BinaryAssetsDirectory: tempDir,
 		Scheme:                buildScheme(),

k3k-kubelet/main.go

@@ -43,8 +43,8 @@ func main() {
 		RunE: run,
 	}
 
-	rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Enable debug logging")
-	rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "json", "Log format (json or console)")
+	rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "", false, "Enable debug logging")
+	rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "text", "Log format (text or json)")
 	rootCmd.PersistentFlags().StringVar(&cfg.ClusterName, "cluster-name", "", "Name of the k3k cluster")
 	rootCmd.PersistentFlags().StringVar(&cfg.ClusterNamespace, "cluster-namespace", "", "Namespace of the k3k cluster")
 	rootCmd.PersistentFlags().StringVar(&cfg.Token, "token", "", "K3S token of the k3k cluster")

main.go

@@ -62,8 +62,8 @@ func main() {
 		RunE: run,
 	}
 
-	rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Debug level logging")
-	rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "json", "Log format (json or console)")
+	rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "", false, "Debug level logging")
+	rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "text", "Log format (text or json)")
 	rootCmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", "", "kubeconfig path")
 	rootCmd.PersistentFlags().StringVar(&config.ClusterCIDR, "cluster-cidr", "", "Cluster CIDR to be added to the networkpolicy")
 	rootCmd.PersistentFlags().StringVar(&config.SharedAgentImage, "agent-shared-image", "rancher/k3k-kubelet", "K3K Virtual Kubelet image")

pkg/apis/k3k.io/v1beta1/types.go

@@ -103,7 +103,7 @@ type ClusterSpec struct {
 	// Expose specifies options for exposing the API server.
 	// By default, it's only exposed as a ClusterIP.
 	//
-	// +kubebuilder:validation:XValidation:rule="[has(self.ingress), has(self.loadbalancer), has(self.nodePort)].filter(x, x).size() <= 1",message="ingress, loadbalancer and nodePort are mutually exclusive; only one can be set"
+	// +kubebuilder:validation:XValidation:rule="[has(self.ingress), has(self.loadBalancer), has(self.nodePort)].filter(x, x).size() <= 1",message="ingress, loadbalancer and nodePort are mutually exclusive; only one can be set"
 	// +optional
 	Expose *ExposeConfig `json:"expose,omitempty"`
 
@@ -176,7 +176,7 @@ type ClusterSpec struct {
 	// CustomCAs specifies the cert/key pairs for custom CA certificates.
 	//
 	// +optional
-	CustomCAs CustomCAs `json:"customCAs,omitempty"`
+	CustomCAs *CustomCAs `json:"customCAs,omitempty"`
 
 	// Sync specifies the resources types that will be synced from virtual cluster to host cluster.
 	//
@@ -190,32 +190,40 @@ type SyncConfig struct {
 	// Services resources sync configuration.
 	//
 	// +kubebuilder:default={"enabled": true}
-	Services ServiceSyncConfig `json:"services,omitempty"`
+	// +optional
+	Services ServiceSyncConfig `json:"services"`
 	// ConfigMaps resources sync configuration.
 	//
 	// +kubebuilder:default={"enabled": true}
-	ConfigMaps ConfigMapSyncConfig `json:"configmaps,omitempty"`
+	// +optional
+	ConfigMaps ConfigMapSyncConfig `json:"configMaps"`
 	// Secrets resources sync configuration.
 	//
 	// +kubebuilder:default={"enabled": true}
-	Secrets SecretSyncConfig `json:"secrets,omitempty"`
+	// +optional
+	Secrets SecretSyncConfig `json:"secrets"`
 	// Ingresses resources sync configuration.
 	//
 	// +kubebuilder:default={"enabled": false}
-	Ingresses IngressSyncConfig `json:"ingresses,omitempty"`
+	// +optional
+	Ingresses IngressSyncConfig `json:"ingresses"`
 	// PersistentVolumeClaims resources sync configuration.
 	//
 	// +kubebuilder:default={"enabled": true}
-	PersistentVolumeClaims PersistentVolumeClaimSyncConfig `json:"persistentVolumeClaims,omitempty"`
+	// +optional
+	PersistentVolumeClaims PersistentVolumeClaimSyncConfig `json:"persistentVolumeClaims"`
 	// PriorityClasses resources sync configuration.
 	//
 	// +kubebuilder:default={"enabled": false}
-	PriorityClasses PriorityClassSyncConfig `json:"priorityClasses,omitempty"`
+	// +optional
+	PriorityClasses PriorityClassSyncConfig `json:"priorityClasses"`
 }
 
 // SecretSyncConfig specifies the sync options for services.
 type SecretSyncConfig struct {
 	// Enabled is an on/off switch for syncing resources.
+	//
+	// +kubebuilder:default=true
 	// +optional
 	Enabled bool `json:"enabled,omitempty"`
 
@@ -229,8 +237,10 @@ type SecretSyncConfig struct {
 // ServiceSyncConfig specifies the sync options for services.
 type ServiceSyncConfig struct {
 	// Enabled is an on/off switch for syncing resources.
-	// +optional
-	Enabled bool `json:"enabled,omitempty"`
+	//
+	// +kubebuilder:default=true
+	// +required
+	Enabled bool `json:"enabled"`
 
 	// Selector specifies set of labels of the resources that will be synced, if empty
 	// then all resources of the given type will be synced.
@@ -242,8 +252,10 @@ type ServiceSyncConfig struct {
 // ConfigMapSyncConfig specifies the sync options for services.
 type ConfigMapSyncConfig struct {
 	// Enabled is an on/off switch for syncing resources.
-	// +optional
-	Enabled bool `json:"enabled,omitempty"`
+	//
+	// +kubebuilder:default=true
+	// +required
+	Enabled bool `json:"enabled"`
 
 	// Selector specifies set of labels of the resources that will be synced, if empty
 	// then all resources of the given type will be synced.
@@ -255,8 +267,10 @@ type ConfigMapSyncConfig struct {
 // IngressSyncConfig specifies the sync options for services.
 type IngressSyncConfig struct {
 	// Enabled is an on/off switch for syncing resources.
-	// +optional
-	Enabled bool `json:"enabled,omitempty"`
+	//
+	// +kubebuilder:default=false
+	// +required
+	Enabled bool `json:"enabled"`
 
 	// Selector specifies set of labels of the resources that will be synced, if empty
 	// then all resources of the given type will be synced.
@@ -268,8 +282,10 @@ type IngressSyncConfig struct {
 // PersistentVolumeClaimSyncConfig specifies the sync options for services.
 type PersistentVolumeClaimSyncConfig struct {
 	// Enabled is an on/off switch for syncing resources.
-	// +optional
-	Enabled bool `json:"enabled,omitempty"`
+	//
+	// +kubebuilder:default=true
+	// +required
+	Enabled bool `json:"enabled"`
 
 	// Selector specifies set of labels of the resources that will be synced, if empty
 	// then all resources of the given type will be synced.
@@ -281,8 +297,10 @@ type PersistentVolumeClaimSyncConfig struct {
 // PriorityClassSyncConfig specifies the sync options for services.
 type PriorityClassSyncConfig struct {
 	// Enabled is an on/off switch for syncing resources.
-	// +optional
-	Enabled bool `json:"enabled,omitempty"`
+	//
+	// +kubebuilder:default=false
+	// +required
+	Enabled bool `json:"enabled"`
 
 	// Selector specifies set of labels of the resources that will be synced, if empty
 	// then all resources of the given type will be synced.
@@ -358,7 +376,7 @@ type ExposeConfig struct {
 	// LoadBalancer specifies options for exposing the API server through a LoadBalancer service.
 	//
 	// +optional
-	LoadBalancer *LoadBalancerConfig `json:"loadbalancer,omitempty"`
+	LoadBalancer *LoadBalancerConfig `json:"loadBalancer,omitempty"`
 
 	// NodePort specifies options for exposing the API server through NodePort.
 	//
@@ -416,32 +434,34 @@ type NodePortConfig struct {
 // CustomCAs specifies the cert/key pairs for custom CA certificates.
 type CustomCAs struct {
 	// Enabled toggles this feature on or off.
-	Enabled bool `json:"enabled,omitempty"`
+	//
+	// +kubebuilder:default=true
+	Enabled bool `json:"enabled"`
 
 	// Sources defines the sources for all required custom CA certificates.
-	Sources CredentialSources `json:"sources,omitempty"`
+	Sources CredentialSources `json:"sources"`
 }
 
 // CredentialSources lists all the required credentials, including both
 // TLS key pairs and single signing keys.
 type CredentialSources struct {
 	// ServerCA specifies the server-ca cert/key pair.
-	ServerCA CredentialSource `json:"serverCA,omitempty"`
+	ServerCA CredentialSource `json:"serverCA"`
 
 	// ClientCA specifies the client-ca cert/key pair.
-	ClientCA CredentialSource `json:"clientCA,omitempty"`
+	ClientCA CredentialSource `json:"clientCA"`
 
 	// RequestHeaderCA specifies the request-header-ca cert/key pair.
-	RequestHeaderCA CredentialSource `json:"requestHeaderCA,omitempty"`
+	RequestHeaderCA CredentialSource `json:"requestHeaderCA"`
 
 	// ETCDServerCA specifies the etcd-server-ca cert/key pair.
-	ETCDServerCA CredentialSource `json:"etcdServerCA,omitempty"`
+	ETCDServerCA CredentialSource `json:"etcdServerCA"`
 
 	// ETCDPeerCA specifies the etcd-peer-ca cert/key pair.
-	ETCDPeerCA CredentialSource `json:"etcdPeerCA,omitempty"`
+	ETCDPeerCA CredentialSource `json:"etcdPeerCA"`
 
 	// ServiceAccountToken specifies the service-account-token key.
-	ServiceAccountToken CredentialSource `json:"serviceAccountToken,omitempty"`
+	ServiceAccountToken CredentialSource `json:"serviceAccountToken"`
 }
 
 // CredentialSource defines where to get a credential from.
@@ -451,8 +471,7 @@ type CredentialSource struct {
 	// The controller expects specific keys inside based on the credential type:
 	// - For TLS pairs (e.g., ServerCA): 'tls.crt' and 'tls.key'.
 	// - For ServiceAccountTokenKey: 'tls.key'.
-	// +optional
-	SecretName string `json:"secretName,omitempty"`
+	SecretName string `json:"secretName"`
 }
 
 // ClusterStatus reflects the observed state of a Cluster.

pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go

@@ -163,7 +163,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 			(*out)[key] = val.DeepCopy()
 		}
 	}
-	out.CustomCAs = in.CustomCAs
+	if in.CustomCAs != nil {
+		in, out := &in.CustomCAs, &out.CustomCAs
+		*out = new(CustomCAs)
+		**out = **in
+	}
 	if in.Sync != nil {
 		in, out := &in.Sync, &out.Sync
 		*out = new(SyncConfig)

pkg/controller/cluster/agent/agent.go

@@ -42,11 +42,8 @@ func configSecretName(clusterName string) string {
 }
 
 func ensureObject(ctx context.Context, cfg *Config, obj ctrlruntimeclient.Object) error {
-	log := ctrl.LoggerFrom(ctx)
-
 	key := ctrlruntimeclient.ObjectKeyFromObject(obj)
-
-	log.Info(fmt.Sprintf("ensuring %T", obj), "key", key)
+	log := ctrl.LoggerFrom(ctx).WithValues("key", key)
 
 	if err := controllerutil.SetControllerReference(cfg.cluster, obj, cfg.scheme); err != nil {
 		return err
@@ -54,11 +51,15 @@ func ensureObject(ctx context.Context, cfg *Config, obj ctrlruntimeclient.Object
 
 	if err := cfg.client.Create(ctx, obj); err != nil {
 		if apierrors.IsAlreadyExists(err) {
+			log.V(1).Info(fmt.Sprintf("Resource %T already exists, updating.", obj))
+
 			return cfg.client.Update(ctx, obj)
 		}
 
 		return err
 	}
 
+	log.V(1).Info(fmt.Sprintf("Creating %T.", obj))
+
 	return nil
 }

pkg/controller/cluster/cluster.go

@@ -161,10 +161,8 @@ func namespaceEventHandler(r *ClusterReconciler) handler.Funcs {
 }
 
 func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
-	log := ctrl.LoggerFrom(ctx).WithValues("cluster", req.NamespacedName)
-	ctx = ctrl.LoggerInto(ctx, log) // enrich the current logger
-
-	log.Info("reconciling cluster")
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("Reconciling Cluster")
 
 	var cluster v1beta1.Cluster
 	if err := c.Client.Get(ctx, req.NamespacedName, &cluster); err != nil {
@@ -178,6 +176,8 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 
 	// Set initial status if not already set
 	if cluster.Status.Phase == "" || cluster.Status.Phase == v1beta1.ClusterUnknown {
+		log.V(1).Info("Updating Cluster status phase")
+
 		cluster.Status.Phase = v1beta1.ClusterProvisioning
 		meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
 			Type:    ConditionReady,
@@ -195,6 +195,8 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 
 	// add finalizer
 	if controllerutil.AddFinalizer(&cluster, clusterFinalizerName) {
+		log.V(1).Info("Updating Cluster adding finalizer")
+
 		if err := c.Client.Update(ctx, &cluster); err != nil {
 			return reconcile.Result{}, err
 		}
@@ -207,6 +209,8 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	reconcilerErr := c.reconcileCluster(ctx, &cluster)
 
 	if !equality.Semantic.DeepEqual(orig.Status, cluster.Status) {
+		log.Info("Updating Cluster status")
+
 		if err := c.Client.Status().Update(ctx, &cluster); err != nil {
 			return reconcile.Result{}, err
 		}
@@ -215,7 +219,7 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	// if there was an error during the reconciliation, return
 	if reconcilerErr != nil {
 		if errors.Is(reconcilerErr, bootstrap.ErrServerNotReady) {
-			log.Info("server not ready, requeueing")
+			log.V(1).Info("Server not ready, requeueing")
 			return reconcile.Result{RequeueAfter: time.Second * 10}, nil
 		}
 
@@ -224,6 +228,8 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 
 	// update Cluster if needed
 	if !equality.Semantic.DeepEqual(orig.Spec, cluster.Spec) {
+		log.Info("Updating Cluster")
+
 		if err := c.Client.Update(ctx, &cluster); err != nil {
 			return reconcile.Result{}, err
 		}
@@ -234,7 +240,7 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 
 func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1beta1.Cluster) error {
 	err := c.reconcile(ctx, cluster)
-	c.updateStatus(cluster, err)
+	c.updateStatus(ctx, cluster, err)
 
 	return err
 }
@@ -264,7 +270,7 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
 	// if the Version is not specified we will try to use the same Kubernetes version of the host.
 	// This version is stored in the Status object, and it will not be updated if already set.
 	if cluster.Spec.Version == "" && cluster.Status.HostVersion == "" {
-		log.Info("cluster version not set")
+		log.V(1).Info("Cluster version not set. Using host version.")
 
 		hostVersion, err := c.DiscoveryClient.ServerVersion()
 		if err != nil {
@@ -295,7 +301,7 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
 	if cluster.Status.ServiceCIDR == "" {
 		// in shared mode try to lookup the serviceCIDR
 		if cluster.Spec.Mode == v1beta1.SharedClusterMode {
-			log.Info("looking up Service CIDR for shared mode")
+			log.V(1).Info("Looking up Service CIDR for shared mode")
 
 			cluster.Status.ServiceCIDR, err = c.lookupServiceCIDR(ctx)
 			if err != nil {
@@ -307,7 +313,7 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
 
 		// in virtual mode assign a default serviceCIDR
 		if cluster.Spec.Mode == v1beta1.VirtualClusterMode {
-			log.Info("assign default service CIDR for virtual mode")
+			log.V(1).Info("assign default service CIDR for virtual mode")
 
 			cluster.Status.ServiceCIDR = defaultVirtualServiceCIDR
 		}
@@ -354,7 +360,7 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
 // ensureBootstrapSecret will create or update the Secret containing the bootstrap data from the k3s server
 func (c *ClusterReconciler) ensureBootstrapSecret(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("ensuring bootstrap secret")
+	log.V(1).Info("Ensuring bootstrap secret")
 
 	bootstrapData, err := bootstrap.GenerateBootstrapData(ctx, cluster, serviceIP, token)
 	if err != nil {
@@ -386,7 +392,7 @@ func (c *ClusterReconciler) ensureBootstrapSecret(ctx context.Context, cluster *
 // ensureKubeconfigSecret will create or update the Secret containing the kubeconfig data from the k3s server
 func (c *ClusterReconciler) ensureKubeconfigSecret(ctx context.Context, cluster *v1beta1.Cluster, serviceIP string, port int) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("ensuring kubeconfig secret")
+	log.V(1).Info("Ensuring Kubeconfig Secret")
 
 	adminKubeconfig := kubeconfig.New()
 
@@ -460,7 +466,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
 
 func (c *ClusterReconciler) ensureNetworkPolicy(ctx context.Context, cluster *v1beta1.Cluster) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("ensuring network policy")
+	log.V(1).Info("Ensuring network policy")
 
 	networkPolicyName := controller.SafeConcatNameWithPrefix(cluster.Name)
 
@@ -544,7 +550,7 @@ func (c *ClusterReconciler) ensureNetworkPolicy(ctx context.Context, cluster *v1
 
 	key := client.ObjectKeyFromObject(currentNetworkPolicy)
 	if result != controllerutil.OperationResultNone {
-		log.Info("cluster network policy updated", "key", key, "result", result)
+		log.V(1).Info("Cluster network policy updated", "key", key, "result", result)
 	}
 
 	return nil
@@ -552,7 +558,7 @@ func (c *ClusterReconciler) ensureNetworkPolicy(ctx context.Context, cluster *v1
 
 func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v1beta1.Cluster) (*v1.Service, error) {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("ensuring cluster service")
+	log.V(1).Info("Ensuring Cluster Service")
 
 	expectedService := server.Service(cluster)
 	currentService := expectedService.DeepCopy()
@@ -572,7 +578,7 @@ func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v
 
 	key := client.ObjectKeyFromObject(currentService)
 	if result != controllerutil.OperationResultNone {
-		log.Info("cluster service updated", "key", key, "result", result)
+		log.V(1).Info("Cluster service updated", "key", key, "result", result)
 	}
 
 	return currentService, nil
@@ -580,7 +586,7 @@ func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v
 
 func (c *ClusterReconciler) ensureIngress(ctx context.Context, cluster *v1beta1.Cluster) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("ensuring cluster ingress")
+	log.V(1).Info("Ensuring cluster ingress")
 
 	expectedServerIngress := server.Ingress(ctx, cluster)
 
@@ -608,7 +614,7 @@ func (c *ClusterReconciler) ensureIngress(ctx context.Context, cluster *v1beta1.
 
 	key := client.ObjectKeyFromObject(currentServerIngress)
 	if result != controllerutil.OperationResultNone {
-		log.Info("cluster ingress updated", "key", key, "result", result)
+		log.V(1).Info("Cluster ingress updated", "key", key, "result", result)
 	}
 
 	return nil
@@ -650,7 +656,7 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1beta1.Cluster
 
 	if result != controllerutil.OperationResultNone {
 		key := client.ObjectKeyFromObject(currentServerStatefulSet)
-		log.Info("ensuring serverStatefulSet", "key", key, "result", result)
+		log.V(1).Info("Ensuring server StatefulSet", "key", key, "result", result)
 	}
 
 	return err
@@ -730,8 +736,8 @@ func (c *ClusterReconciler) validate(cluster *v1beta1.Cluster, policy v1beta1.Vi
 		return fmt.Errorf("%w: mode %q is not allowed by the policy %q", ErrClusterValidation, cluster.Spec.Mode, policy.Name)
 	}
 
-	if cluster.Spec.CustomCAs.Enabled {
-		if err := c.validateCustomCACerts(cluster); err != nil {
+	if cluster.Spec.CustomCAs != nil && cluster.Spec.CustomCAs.Enabled {
+		if err := c.validateCustomCACerts(cluster.Spec.CustomCAs.Sources); err != nil {
 			return fmt.Errorf("%w: %w", ErrClusterValidation, err)
 		}
 	}
@@ -753,7 +759,7 @@ func (c *ClusterReconciler) lookupServiceCIDR(ctx context.Context) (string, erro
 	// Try to look for the serviceCIDR creating a failing service.
 	// The error should contain the expected serviceCIDR
 
-	log.Info("looking up serviceCIDR from a failing service creation")
+	log.V(1).Info("Looking up Service CIDR from a failing service creation")
 
 	failingSvc := v1.Service{
 		ObjectMeta: metav1.ObjectMeta{Name: "fail", Namespace: "default"},
@@ -765,7 +771,7 @@ func (c *ClusterReconciler) lookupServiceCIDR(ctx context.Context) (string, erro
 
 		if len(splittedErrMsg) > 1 {
 			serviceCIDR := strings.TrimSpace(splittedErrMsg[1])
-			log.Info("found serviceCIDR from failing service creation: " + serviceCIDR)
+			log.V(1).Info("Found Service CIDR from failing service creation: " + serviceCIDR)
 
 			// validate serviceCIDR
 			_, serviceCIDRAddr, err := net.ParseCIDR(serviceCIDR)
@@ -779,7 +785,7 @@ func (c *ClusterReconciler) lookupServiceCIDR(ctx context.Context) (string, erro
 
 	// Try to look for the the kube-apiserver Pod, and look for the '--service-cluster-ip-range' flag.
 
-	log.Info("looking up serviceCIDR from kube-apiserver pod")
+	log.V(1).Info("Looking up Service CIDR from kube-apiserver pod")
 
 	matchingLabels := client.MatchingLabels(map[string]string{
 		"component": "kube-apiserver",
@@ -802,12 +808,12 @@ func (c *ClusterReconciler) lookupServiceCIDR(ctx context.Context) (string, erro
 		for _, arg := range apiServerArgs {
 			if strings.HasPrefix(arg, "--service-cluster-ip-range=") {
 				serviceCIDR := strings.TrimPrefix(arg, "--service-cluster-ip-range=")
-				log.Info("found serviceCIDR from kube-apiserver pod: " + serviceCIDR)
+				log.V(1).Info("Found Service CIDR from kube-apiserver pod: " + serviceCIDR)
 
 				// validate serviceCIDR
 				_, serviceCIDRAddr, err := net.ParseCIDR(serviceCIDR)
 				if err != nil {
-					log.Error(err, "serviceCIDR is not valid")
+					log.Error(err, "Service CIDR is not valid")
 					break
 				}
 
@@ -822,8 +828,7 @@ func (c *ClusterReconciler) lookupServiceCIDR(ctx context.Context) (string, erro
 }
 
 // validateCustomCACerts will make sure that all the cert secrets exists
-func (c *ClusterReconciler) validateCustomCACerts(cluster *v1beta1.Cluster) error {
-	credentialSources := cluster.Spec.CustomCAs.Sources
+func (c *ClusterReconciler) validateCustomCACerts(credentialSources v1beta1.CredentialSources) error {
 	if credentialSources.ClientCA.SecretName == "" ||
 		credentialSources.ServerCA.SecretName == "" ||
 		credentialSources.ETCDPeerCA.SecretName == "" ||

pkg/controller/cluster/cluster_finalize.go

@@ -12,7 +12,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
+	coordinationv1 "k8s.io/api/coordination/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 
@@ -23,7 +25,7 @@ import (
 
 func (c *ClusterReconciler) finalizeCluster(ctx context.Context, cluster *v1beta1.Cluster) (reconcile.Result, error) {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("finalizing Cluster")
+	log.V(1).Info("Deleting Cluster")
 
 	// Set the Terminating phase and condition
 	cluster.Status.Phase = v1beta1.ClusterTerminating
@@ -40,7 +42,7 @@ func (c *ClusterReconciler) finalizeCluster(ctx context.Context, cluster *v1beta
 
 	// Deallocate ports for kubelet and webhook if used
 	if cluster.Spec.Mode == v1beta1.SharedClusterMode && cluster.Spec.MirrorHostNodes {
-		log.Info("dellocating ports for kubelet and webhook")
+		log.V(1).Info("dellocating ports for kubelet and webhook")
 
 		if err := c.PortAllocator.DeallocateKubeletPort(ctx, cluster.Name, cluster.Namespace, cluster.Status.KubeletPort); err != nil {
 			return reconcile.Result{}, err
@@ -51,8 +53,25 @@ func (c *ClusterReconciler) finalizeCluster(ctx context.Context, cluster *v1beta
 		}
 	}
 
+	// delete API server lease
+	lease := &coordinationv1.Lease{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Lease",
+			APIVersion: "coordination.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cluster.Name,
+			Namespace: cluster.Namespace,
+		},
+	}
+	if err := c.Client.Delete(ctx, lease); err != nil && !apierrors.IsNotFound(err) {
+		return reconcile.Result{}, err
+	}
+
 	// Remove finalizer from the cluster and update it only when all resources are cleaned up
 	if controllerutil.RemoveFinalizer(cluster, clusterFinalizerName) {
+		log.Info("Deleting Cluster removing finalizer")
+
 		if err := c.Client.Update(ctx, cluster); err != nil {
 			return reconcile.Result{}, err
 		}
@@ -62,6 +81,9 @@ func (c *ClusterReconciler) finalizeCluster(ctx context.Context, cluster *v1beta
 }
 
 func (c *ClusterReconciler) unbindClusterRoles(ctx context.Context, cluster *v1beta1.Cluster) error {
+	log := ctrl.LoggerFrom(ctx)
+	log.V(1).Info("Unbinding ClusterRoles")
+
 	clusterRoles := []string{"k3k-kubelet-node", "k3k-priorityclass"}
 
 	var err error

pkg/controller/cluster/cluster_suite_test.go

@@ -41,7 +41,7 @@ var (
 var _ = BeforeSuite(func() {
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")},
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")},
 		ErrorIfCRDPathMissing: true,
 	}
 

pkg/controller/cluster/cluster_test.go

@@ -53,6 +53,19 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
 				Expect(cluster.Spec.Servers).To(Equal(ptr.To[int32](1)))
 				Expect(cluster.Spec.Version).To(BeEmpty())
 
+				Expect(cluster.Spec.CustomCAs).To(BeNil())
+
+				// sync
+				// enabled by default
+				Expect(cluster.Spec.Sync).To(Not(BeNil()))
+				Expect(cluster.Spec.Sync.ConfigMaps.Enabled).To(BeTrue())
+				Expect(cluster.Spec.Sync.PersistentVolumeClaims.Enabled).To(BeTrue())
+				Expect(cluster.Spec.Sync.Secrets.Enabled).To(BeTrue())
+				Expect(cluster.Spec.Sync.Services.Enabled).To(BeTrue())
+				// disabled by default
+				Expect(cluster.Spec.Sync.Ingresses.Enabled).To(BeFalse())
+				Expect(cluster.Spec.Sync.PriorityClasses.Enabled).To(BeFalse())
+
 				Expect(cluster.Spec.Persistence.Type).To(Equal(v1beta1.DynamicPersistenceMode))
 				Expect(cluster.Spec.Persistence.StorageRequestSize).To(Equal("2G"))
 

pkg/controller/cluster/pod.go

@@ -9,7 +9,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	v1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -44,14 +43,10 @@ func AddPodController(ctx context.Context, mgr manager.Manager, maxConcurrentRec
 
 func (r *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("reconciling pod")
+	log.V(1).Info("Reconciling Pod")
 
 	var pod v1.Pod
 	if err := r.Client.Get(ctx, req.NamespacedName, &pod); err != nil {
-		if !apierrors.IsNotFound(err) {
-			return reconcile.Result{}, err
-		}
-
 		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
 	}
 
@@ -74,6 +69,8 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r
 			},
 		}
 
+		log.V(1).Info("Deleting Virtual Pod", "name", virtName, "namespace", virtNamespace)
+
 		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(virtualClient.Delete(ctx, &virtPod))
 	}
 

pkg/controller/cluster/server/server.go

@@ -13,6 +13,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -268,6 +269,10 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
 	if s.cluster.Spec.Persistence.Type == v1beta1.DynamicPersistenceMode {
 		persistent = true
 		pvClaim = s.setupDynamicPersistence()
+
+		if err := controllerutil.SetControllerReference(s.cluster, &pvClaim, s.client.Scheme()); err != nil {
+			return nil, err
+		}
 	}
 
 	var (
@@ -330,7 +335,7 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
 		volumeMounts = append(volumeMounts, volumeMount)
 	}
 
-	if s.cluster.Spec.CustomCAs.Enabled {
+	if s.cluster.Spec.CustomCAs != nil && s.cluster.Spec.CustomCAs.Enabled {
 		vols, mounts, err := s.loadCACertBundle(ctx)
 		if err != nil {
 			return nil, err
@@ -434,6 +439,10 @@ func (s *Server) setupStartCommand() (string, error) {
 }
 
 func (s *Server) loadCACertBundle(ctx context.Context) ([]v1.Volume, []v1.VolumeMount, error) {
+	if s.cluster.Spec.CustomCAs == nil {
+		return nil, nil, fmt.Errorf("customCAs not found")
+	}
+
 	customCerts := s.cluster.Spec.CustomCAs.Sources
 	caCertMap := map[string]string{
 		"server-ca":         customCerts.ServerCA.SecretName,

pkg/controller/cluster/service.go

@@ -39,7 +39,7 @@ func AddServiceController(ctx context.Context, mgr manager.Manager, maxConcurren
 
 func (r *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("ensuring service status to virtual cluster")
+	log.V(1).Info("Reconciling Service")
 
 	var hostService v1.Service
 	if err := r.HostClient.Get(ctx, req.NamespacedName, &hostService); err != nil {
@@ -53,7 +53,7 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	virtualServiceNamespace, virtualServiceNamespaceFound := hostService.Annotations[translate.ResourceNamespaceAnnotation]
 
 	if !virtualServiceNameFound || !virtualServiceNamespaceFound {
-		log.V(1).Info(fmt.Sprintf("service %s/%s does not have virtual service annotations, skipping", hostService.Namespace, hostService.Name))
+		log.V(1).Info(fmt.Sprintf("Service %s/%s does not have virtual service annotations, skipping", hostService.Namespace, hostService.Name))
 		return reconcile.Result{}, nil
 	}
 
@@ -80,7 +80,10 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	}
 
 	if !equality.Semantic.DeepEqual(virtualService.Status.LoadBalancer, hostService.Status.LoadBalancer) {
+		log.V(1).Info("Updating Virtual Service Status", "name", virtualServiceName, "namespace", virtualServiceNamespace)
+
 		virtualService.Status.LoadBalancer = hostService.Status.LoadBalancer
+
 		if err := virtualClient.Status().Update(ctx, &virtualService); err != nil {
 			return reconcile.Result{}, err
 		}

pkg/controller/cluster/statefulset.go

@@ -62,7 +62,7 @@ func AddStatefulSetController(ctx context.Context, mgr manager.Manager, maxConcu
 
 func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("reconciling statefulset")
+	log.Info("Reconciling StatefulSet")
 
 	var sts apps.StatefulSet
 	if err := p.Client.Get(ctx, req.NamespacedName, &sts); err != nil {
@@ -116,10 +116,12 @@ func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Req
 
 func (p *StatefulSetReconciler) handleServerPod(ctx context.Context, cluster v1beta1.Cluster, pod *v1.Pod) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("handling server pod")
+	log.V(1).Info("Handling Server Pod")
 
 	if pod.DeletionTimestamp.IsZero() {
 		if controllerutil.AddFinalizer(pod, etcdPodFinalizerName) {
+			log.V(1).Info("Server Pod is being deleted. Removing finalizer", "pod", pod.Name, "namespace", pod.Namespace)
+
 			return p.Client.Update(ctx, pod)
 		}
 
@@ -131,6 +133,8 @@ func (p *StatefulSetReconciler) handleServerPod(ctx context.Context, cluster v1b
 	// check if cluster is deleted then remove the finalizer from the pod
 	if cluster.Name == "" {
 		if controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName) {
+			log.V(1).Info("Cluster was deleted. Deleting Server Pod removing finalizer", "pod", pod.Name, "namespace", pod.Namespace)
+
 			if err := p.Client.Update(ctx, pod); err != nil {
 				return err
 			}
@@ -161,6 +165,8 @@ func (p *StatefulSetReconciler) handleServerPod(ctx context.Context, cluster v1b
 
 	// remove our finalizer from the list and update it.
 	if controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName) {
+		log.V(1).Info("Deleting Server Pod removing finalizer", "pod", pod.Name, "namespace", pod.Namespace)
+
 		if err := p.Client.Update(ctx, pod); err != nil {
 			return err
 		}
@@ -171,7 +177,7 @@ func (p *StatefulSetReconciler) handleServerPod(ctx context.Context, cluster v1b
 
 func (p *StatefulSetReconciler) getETCDTLS(ctx context.Context, cluster *v1beta1.Cluster) (*tls.Config, error) {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("generating etcd TLS client certificate", "cluster", cluster)
+	log.V(1).Info("Generating ETCD TLS client certificate", "cluster", cluster)
 
 	token, err := p.clusterToken(ctx, cluster)
 	if err != nil {
@@ -219,7 +225,7 @@ func (p *StatefulSetReconciler) getETCDTLS(ctx context.Context, cluster *v1beta1
 // removePeer removes a peer from the cluster. The peer name and IP address must both match.
 func removePeer(ctx context.Context, client *clientv3.Client, name, address string) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("removing peer from cluster", "name", name, "address", address)
+	log.V(1).Info("Removing peer from cluster", "name", name, "address", address)
 
 	ctx, cancel := context.WithTimeout(ctx, memberRemovalTimeout)
 	defer cancel()
@@ -241,7 +247,7 @@ func removePeer(ctx context.Context, client *clientv3.Client, name, address stri
 			}
 
 			if u.Hostname() == address {
-				log.Info("removing member from etcd", "name", member.Name, "id", member.ID, "address", address)
+				log.V(1).Info("Removing member from ETCD", "name", member.Name, "id", member.ID, "address", address)
 
 				_, err := client.MemberRemove(ctx, member.ID)
 				if errors.Is(err, rpctypes.ErrGRPCMemberNotFound) {
@@ -280,6 +286,8 @@ func (p *StatefulSetReconciler) clusterToken(ctx context.Context, cluster *v1bet
 }
 
 func (p *StatefulSetReconciler) handleDeletion(ctx context.Context, sts *apps.StatefulSet) (ctrl.Result, error) {
+	log := ctrl.LoggerFrom(ctx)
+
 	podList, err := p.listPods(ctx, sts)
 	if err != nil {
 		return reconcile.Result{}, err
@@ -287,6 +295,8 @@ func (p *StatefulSetReconciler) handleDeletion(ctx context.Context, sts *apps.St
 
 	for _, pod := range podList.Items {
 		if controllerutil.RemoveFinalizer(&pod, etcdPodFinalizerName) {
+			log.V(1).Info("Updating Server Pod removing finalizer", "name", pod.Name, "namespace", pod.Namespace)
+
 			if err := p.Client.Update(ctx, &pod); err != nil {
 				return reconcile.Result{}, err
 			}

pkg/controller/cluster/status.go

@@ -1,12 +1,14 @@
 package cluster
 
 import (
+	"context"
 	"errors"
 
 	"k8s.io/apimachinery/pkg/api/meta"
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
@@ -24,7 +26,10 @@ const (
 	ReasonTerminating        = "Terminating"
 )
 
-func (c *ClusterReconciler) updateStatus(cluster *v1beta1.Cluster, reconcileErr error) {
+func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.Cluster, reconcileErr error) {
+	log := ctrl.LoggerFrom(ctx)
+	log.V(1).Info("Updating Cluster Conditions")
+
 	if !cluster.DeletionTimestamp.IsZero() {
 		cluster.Status.Phase = v1beta1.ClusterTerminating
 		meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{

pkg/controller/cluster/token.go

@@ -62,7 +62,7 @@ func (c *ClusterReconciler) ensureTokenSecret(ctx context.Context, cluster *v1be
 		return string(tokenSecret.Data["token"]), nil
 	}
 
-	log.Info("Token secret is not specified, creating a random token")
+	log.V(1).Info("Token secret is not specified, creating a random token")
 
 	token, err := random(16)
 	if err != nil {
@@ -77,7 +77,7 @@ func (c *ClusterReconciler) ensureTokenSecret(ctx context.Context, cluster *v1be
 	})
 
 	if result != controllerutil.OperationResultNone {
-		log.Info("ensuring tokenSecret", "key", key, "result", result)
+		log.V(1).Info("Ensuring tokenSecret", "key", key, "result", result)
 	}
 
 	return token, err

pkg/controller/kubeconfig/kubeconfig.go

@@ -108,13 +108,27 @@ func getURLFromService(ctx context.Context, client client.Client, cluster *v1bet
 	ip := k3kService.Spec.ClusterIP
 	port := int32(443)
 
+	if len(k3kService.Spec.Ports) == 0 {
+		logrus.Warn("No ports exposed by the cluster service.")
+	}
+
 	switch k3kService.Spec.Type {
 	case v1.ServiceTypeNodePort:
 		ip = hostServerIP
-		port = k3kService.Spec.Ports[0].NodePort
+
+		if len(k3kService.Spec.Ports) > 0 {
+			port = k3kService.Spec.Ports[0].NodePort
+		}
 	case v1.ServiceTypeLoadBalancer:
-		ip = k3kService.Status.LoadBalancer.Ingress[0].IP
-		port = k3kService.Spec.Ports[0].Port
+		if len(k3kService.Status.LoadBalancer.Ingress) > 0 {
+			ip = k3kService.Status.LoadBalancer.Ingress[0].IP
+		} else {
+			logrus.Warn("No ingress found in LoadBalancer service.")
+		}
+
+		if len(k3kService.Spec.Ports) > 0 {
+			port = k3kService.Spec.Ports[0].Port
+		}
 	}
 
 	if serverPort != 0 {
@@ -122,7 +136,7 @@ func getURLFromService(ctx context.Context, client client.Client, cluster *v1bet
 	}
 
 	if !slices.Contains(cluster.Status.TLSSANs, ip) {
-		logrus.Warnf("ip %s not in tlsSANs", ip)
+		logrus.Warnf("IP %s not in tlsSANs.", ip)
 
 		if len(cluster.Spec.TLSSANs) > 0 {
 			logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0])
@@ -133,7 +147,7 @@ func getURLFromService(ctx context.Context, client client.Client, cluster *v1bet
 
 			ip = cluster.Status.TLSSANs[0]
 		} else {
-			logrus.Warn("ip not found in tlsSANs. This could cause issue with the certificate validation.")
+			logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.")
 		}
 	}
 

pkg/controller/policy/namespace.go

@@ -17,7 +17,7 @@ import (
 // reconcileNamespacePodSecurityLabels will update the labels of the namespace to reconcile the PSA level specified in the VirtualClusterPolicy
 func (c *VirtualClusterPolicyReconciler) reconcileNamespacePodSecurityLabels(ctx context.Context, namespace *v1.Namespace, policy *v1beta1.VirtualClusterPolicy) {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("reconciling PSA labels")
+	log.V(1).Info("Reconciling PSA labels")
 
 	// cleanup of old labels
 	delete(namespace.Labels, "pod-security.kubernetes.io/enforce")
@@ -44,7 +44,7 @@ func (c *VirtualClusterPolicyReconciler) reconcileNamespacePodSecurityLabels(ctx
 // deleting the resources in them with the "app.kubernetes.io/managed-by=k3k-policy-controller" label
 func (c *VirtualClusterPolicyReconciler) cleanupNamespaces(ctx context.Context) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("deleting resources")
+	log.V(1).Info("Cleanup Namespace resources")
 
 	var namespaces v1.NamespaceList
 	if err := c.Client.List(ctx, &namespaces); err != nil {

pkg/controller/policy/networkpolicy.go

@@ -17,7 +17,7 @@ import (
 
 func (c *VirtualClusterPolicyReconciler) reconcileNetworkPolicy(ctx context.Context, namespace string, policy *v1beta1.VirtualClusterPolicy) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("reconciling NetworkPolicy")
+	log.V(1).Info("Reconciling NetworkPolicy")
 
 	var cidrList []string
 
@@ -46,13 +46,18 @@ func (c *VirtualClusterPolicyReconciler) reconcileNetworkPolicy(ctx context.Cont
 
 	// if disabled then delete the existing network policy
 	if policy.Spec.DisableNetworkPolicy {
-		err := c.Client.Delete(ctx, networkPolicy)
-		return client.IgnoreNotFound(err)
+		log.V(1).Info("Deleting NetworkPolicy")
+
+		return client.IgnoreNotFound(c.Client.Delete(ctx, networkPolicy))
 	}
 
+	log.V(1).Info("Creating NetworkPolicy")
+
 	// otherwise try to create/update
 	err := c.Client.Create(ctx, networkPolicy)
 	if apierrors.IsAlreadyExists(err) {
+		log.V(1).Info("NetworkPolicy already exists, updating.")
+
 		return c.Client.Update(ctx, networkPolicy)
 	}
 

pkg/controller/policy/policy.go

@@ -248,7 +248,7 @@ func clusterEventHandler(r *VirtualClusterPolicyReconciler) handler.Funcs {
 
 func (c *VirtualClusterPolicyReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("reconciling VirtualClusterPolicy")
+	log.Info("Reconciling VirtualClusterPolicy")
 
 	var policy v1beta1.VirtualClusterPolicy
 	if err := c.Client.Get(ctx, req.NamespacedName, &policy); err != nil {
@@ -261,6 +261,8 @@ func (c *VirtualClusterPolicyReconciler) Reconcile(ctx context.Context, req reco
 
 	// update Status if needed
 	if !reflect.DeepEqual(orig.Status, policy.Status) {
+		log.Info("Updating VirtualClusterPolicy Status")
+
 		if err := c.Client.Status().Update(ctx, &policy); err != nil {
 			return reconcile.Result{}, err
 		}
@@ -273,6 +275,8 @@ func (c *VirtualClusterPolicyReconciler) Reconcile(ctx context.Context, req reco
 
 	// update VirtualClusterPolicy if needed
 	if !reflect.DeepEqual(orig, policy) {
+		log.Info("Updating VirtualClusterPolicy")
+
 		if err := c.Client.Update(ctx, &policy); err != nil {
 			return reconcile.Result{}, err
 		}
@@ -295,7 +299,7 @@ func (c *VirtualClusterPolicyReconciler) reconcileVirtualClusterPolicy(ctx conte
 
 func (c *VirtualClusterPolicyReconciler) reconcileMatchingNamespaces(ctx context.Context, policy *v1beta1.VirtualClusterPolicy) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("reconciling matching Namespaces")
+	log.V(1).Info("Reconciling matching Namespaces")
 
 	listOpts := client.MatchingLabels{
 		PolicyNameLabelKey: policy.Name,
@@ -307,8 +311,10 @@ func (c *VirtualClusterPolicyReconciler) reconcileMatchingNamespaces(ctx context
 	}
 
 	for _, ns := range namespaces.Items {
-		ctx = ctrl.LoggerInto(ctx, log.WithValues("namespace", ns.Name))
-		log.Info("reconciling Namespace")
+		log = log.WithValues("namespace", ns.Name)
+		ctx = ctrl.LoggerInto(ctx, log)
+
+		log.V(1).Info("Reconciling Namespace")
 
 		orig := ns.DeepCopy()
 
@@ -331,6 +337,8 @@ func (c *VirtualClusterPolicyReconciler) reconcileMatchingNamespaces(ctx context
 		c.reconcileNamespacePodSecurityLabels(ctx, &ns, policy)
 
 		if !reflect.DeepEqual(orig, &ns) {
+			log.Info("Updating Namespace")
+
 			if err := c.Client.Update(ctx, &ns); err != nil {
 				return err
 			}
@@ -342,7 +350,7 @@ func (c *VirtualClusterPolicyReconciler) reconcileMatchingNamespaces(ctx context
 
 func (c *VirtualClusterPolicyReconciler) reconcileQuota(ctx context.Context, namespace string, policy *v1beta1.VirtualClusterPolicy) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("reconciling ResourceQuota")
+	log.V(1).Info("Reconciling ResourceQuota")
 
 	if policy.Spec.Quota == nil {
 		// check if resourceQuota object exists and deletes it.
@@ -357,6 +365,8 @@ func (c *VirtualClusterPolicyReconciler) reconcileQuota(ctx context.Context, nam
 			return client.IgnoreNotFound(err)
 		}
 
+		log.V(1).Info("Deleting ResourceQuota")
+
 		return c.Client.Delete(ctx, &toDeleteResourceQuota)
 	}
 
@@ -381,8 +391,12 @@ func (c *VirtualClusterPolicyReconciler) reconcileQuota(ctx context.Context, nam
 		return err
 	}
 
+	log.V(1).Info("Creating ResourceQuota")
+
 	err := c.Client.Create(ctx, resourceQuota)
 	if apierrors.IsAlreadyExists(err) {
+		log.V(1).Info("ResourceQuota already exists, updating.")
+
 		return c.Client.Update(ctx, resourceQuota)
 	}
 
@@ -391,7 +405,7 @@ func (c *VirtualClusterPolicyReconciler) reconcileQuota(ctx context.Context, nam
 
 func (c *VirtualClusterPolicyReconciler) reconcileLimit(ctx context.Context, namespace string, policy *v1beta1.VirtualClusterPolicy) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("reconciling LimitRange")
+	log.V(1).Info("Reconciling LimitRange")
 
 	// delete limitrange if spec.limits isnt specified.
 	if policy.Spec.Limit == nil {
@@ -406,6 +420,8 @@ func (c *VirtualClusterPolicyReconciler) reconcileLimit(ctx context.Context, nam
 			return client.IgnoreNotFound(err)
 		}
 
+		log.V(1).Info("Deleting LimitRange")
+
 		return c.Client.Delete(ctx, &toDeleteLimitRange)
 	}
 
@@ -429,8 +445,12 @@ func (c *VirtualClusterPolicyReconciler) reconcileLimit(ctx context.Context, nam
 		return err
 	}
 
+	log.V(1).Info("Creating LimitRange")
+
 	err := c.Client.Create(ctx, limitRange)
 	if apierrors.IsAlreadyExists(err) {
+		log.V(1).Info("LimitRange already exists, updating.")
+
 		return c.Client.Update(ctx, limitRange)
 	}
 
@@ -439,7 +459,7 @@ func (c *VirtualClusterPolicyReconciler) reconcileLimit(ctx context.Context, nam
 
 func (c *VirtualClusterPolicyReconciler) reconcileClusters(ctx context.Context, namespace *v1.Namespace, policy *v1beta1.VirtualClusterPolicy) error {
 	log := ctrl.LoggerFrom(ctx)
-	log.Info("reconciling Clusters")
+	log.V(1).Info("Reconciling Clusters")
 
 	var clusters v1beta1.ClusterList
 	if err := c.Client.List(ctx, &clusters, client.InNamespace(namespace.Name)); err != nil {
@@ -455,6 +475,8 @@ func (c *VirtualClusterPolicyReconciler) reconcileClusters(ctx context.Context,
 		cluster.Spec.NodeSelector = policy.Spec.DefaultNodeSelector
 
 		if !reflect.DeepEqual(orig, cluster) {
+			log.V(1).Info("Updating Cluster", "cluster", cluster.Name, "namespace", namespace.Name)
+
 			// continue updating also the other clusters even if an error occurred
 			clusterUpdateErrs = append(clusterUpdateErrs, c.Client.Update(ctx, &cluster))
 		}

pkg/controller/policy/policy_suite_test.go

@@ -38,7 +38,7 @@ var (
 var _ = BeforeSuite(func() {
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")},
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")},
 		ErrorIfCRDPathMissing: true,
 	}
 	cfg, err := testEnv.Start()

pkg/log/zap.go

@@ -27,7 +27,7 @@ func newEncoder(format string) zapcore.Encoder {
 	encCfg.EncodeTime = zapcore.ISO8601TimeEncoder
 
 	var encoder zapcore.Encoder
-	if format == "console" {
+	if format == "text" {
 		encCfg.EncodeLevel = zapcore.CapitalColorLevelEncoder
 		encoder = zapcore.NewConsoleEncoder(encCfg)
 	} else {

scripts/generate

@@ -10,4 +10,11 @@ CONTROLLER_TOOLS_VERSION=v0.16.0
 go run sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSION} \
     crd:generateEmbeddedObjectMeta=true,allowDangerousTypes=false \
     object paths=./pkg/apis/... \
-    output:crd:dir=./charts/k3k/crds
+    output:crd:dir=./charts/k3k/templates/crds
+
+# add the 'helm.sh/resource-policy: keep' annotation to the CRDs
+for f in ./charts/k3k/templates/crds/*.yaml; do
+    sed -i '0,/^[[:space:]]*annotations:/s/^[[:space:]]*annotations:/&\n    helm.sh\/resource-policy: keep/' "$f"
+    echo "Validating $f"
+    yq . "$f" > /dev/null
+done

tests/cli_test.go

@@ -66,7 +66,7 @@ var _ = When("using the k3kcli", Label("cli"), func() {
 
 			_, stderr, err = K3kcli("cluster", "delete", clusterName)
 			Expect(err).To(Not(HaveOccurred()), string(stderr))
-			Expect(stderr).To(ContainSubstring("Deleting [%s] cluster in namespace [%s]", clusterName, clusterNamespace))
+			Expect(stderr).To(ContainSubstring(`Deleting '%s' cluster in namespace '%s'`, clusterName, clusterNamespace))
 
 			// The deletion could take a bit
 			Eventually(func() string {
@@ -92,7 +92,7 @@ var _ = When("using the k3kcli", Label("cli"), func() {
 
 			_, stderr, err = K3kcli("policy", "create", policyName)
 			Expect(err).To(Not(HaveOccurred()), string(stderr))
-			Expect(stderr).To(ContainSubstring("Creating policy [%s]", policyName))
+			Expect(stderr).To(ContainSubstring(`Creating policy '%s'`, policyName))
 
 			stdout, stderr, err = K3kcli("policy", "list")
 			Expect(err).To(Not(HaveOccurred()), string(stderr))
@@ -102,7 +102,7 @@ var _ = When("using the k3kcli", Label("cli"), func() {
 			stdout, stderr, err = K3kcli("policy", "delete", policyName)
 			Expect(err).To(Not(HaveOccurred()), string(stderr))
 			Expect(stdout).To(BeEmpty())
-			Expect(stderr).To(BeEmpty())
+			Expect(stderr).To(ContainSubstring(`Policy '%s' deleted`, policyName))
 
 			stdout, stderr, err = K3kcli("policy", "list")
 			Expect(err).To(Not(HaveOccurred()), string(stderr))
@@ -140,7 +140,7 @@ var _ = When("using the k3kcli", Label("cli"), func() {
 
 			_, stderr, err = K3kcli("cluster", "delete", clusterName)
 			Expect(err).To(Not(HaveOccurred()), string(stderr))
-			Expect(stderr).To(ContainSubstring("Deleting [%s] cluster in namespace [%s]", clusterName, clusterNamespace))
+			Expect(stderr).To(ContainSubstring(`Deleting '%s' cluster in namespace '%s'`, clusterName, clusterNamespace))
 		})
 	})
 })

tests/cluster_certs_test.go

@@ -13,7 +13,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), func() {
+var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), Label(certificatesTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {
@@ -21,6 +21,10 @@ var _ = When("a cluster with custom certificates is installed with individual ce
 
 		namespace := NewNamespace()
 
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		// create custom cert secret
 		customCertDir := "testdata/customcerts/"
 
@@ -56,7 +60,7 @@ var _ = When("a cluster with custom certificates is installed with individual ce
 
 		cluster := NewCluster(namespace.Name)
 
-		cluster.Spec.CustomCAs = v1beta1.CustomCAs{
+		cluster.Spec.CustomCAs = &v1beta1.CustomCAs{
 			Enabled: true,
 			Sources: v1beta1.CredentialSources{
 				ServerCA: v1beta1.CredentialSource{

tests/cluster_network_test.go

@@ -5,7 +5,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("two virtual clusters are installed", Label("e2e"), func() {
+var _ = When("two virtual clusters are installed", Label("e2e"), Label(networkingTestsLabel), func() {
 	var (
 		cluster1 *VirtualCluster
 		cluster2 *VirtualCluster
@@ -28,7 +28,6 @@ var _ = When("two virtual clusters are installed", Label("e2e"), func() {
 
 		var (
 			stdout  string
-			stderr  string
 			curlCmd string
 			err     error
 		)
@@ -70,25 +69,25 @@ var _ = When("two virtual clusters are installed", Label("e2e"), func() {
 		// Pods in Cluster 1 should not be able to reach the Pod in Cluster 2
 
 		curlCmd = "curl --no-progress-meter " + pod1Cluster2IP
-		_, stderr, err = cluster1.ExecCmd(pod1Cluster1, curlCmd)
+		stdout, _, err = cluster1.ExecCmd(pod1Cluster1, curlCmd)
 		Expect(err).Should(HaveOccurred())
-		Expect(stderr).To(ContainSubstring("Failed to connect"))
+		Expect(stdout).To(Not(ContainSubstring("Welcome to nginx!")))
 
 		curlCmd = "curl --no-progress-meter " + pod1Cluster2IP
-		_, stderr, err = cluster1.ExecCmd(pod2Cluster1, curlCmd)
+		stdout, _, err = cluster1.ExecCmd(pod2Cluster1, curlCmd)
 		Expect(err).To(HaveOccurred())
-		Expect(stderr).To(ContainSubstring("Failed to connect"))
+		Expect(stdout).To(Not(ContainSubstring("Welcome to nginx!")))
 
 		// Pod in Cluster 2 should not be able to reach Pods in Cluster 1
 
 		curlCmd = "curl --no-progress-meter " + pod1Cluster1IP
-		_, stderr, err = cluster2.ExecCmd(pod1Cluster2, curlCmd)
+		stdout, _, err = cluster2.ExecCmd(pod1Cluster2, curlCmd)
 		Expect(err).To(HaveOccurred())
-		Expect(stderr).To(ContainSubstring("Failed to connect"))
+		Expect(stdout).To(Not(ContainSubstring("Welcome to nginx!")))
 
 		curlCmd = "curl --no-progress-meter " + pod2Cluster1IP
-		_, stderr, err = cluster2.ExecCmd(pod1Cluster2, curlCmd)
+		stdout, _, err = cluster2.ExecCmd(pod1Cluster2, curlCmd)
 		Expect(err).To(HaveOccurred())
-		Expect(stderr).To(ContainSubstring("Failed to connect"))
+		Expect(stdout).To(Not(ContainSubstring("Welcome to nginx!")))
 	})
 })

tests/cluster_persistence_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/x509"
 	"errors"
-	"fmt"
 	"time"
 
 	"k8s.io/utils/ptr"
@@ -20,15 +19,15 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("an ephemeral cluster is installed", Label("e2e"), func() {
+var _ = When("an ephemeral cluster is installed", Label("e2e"), Label(persistenceTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {
 		virtualCluster = NewVirtualCluster()
-	})
 
-	AfterEach(func() {
-		DeleteNamespaces(virtualCluster.Cluster.Namespace)
+		DeferCleanup(func() {
+			DeleteNamespaces(virtualCluster.Cluster.Namespace)
+		})
 	})
 
 	It("can create a nginx pod", func() {
@@ -111,7 +110,7 @@ var _ = When("an ephemeral cluster is installed", Label("e2e"), func() {
 	})
 })
 
-var _ = When("a dynamic cluster is installed", Label("e2e"), func() {
+var _ = When("a dynamic cluster is installed", Label("e2e"), Label(persistenceTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {
@@ -154,7 +153,9 @@ var _ = When("a dynamic cluster is installed", Label("e2e"), func() {
 
 		namespace := NewNamespace()
 
-		By(fmt.Sprintf("Creating new virtual cluster in namespace %s", namespace.Name))
+		DeferCleanup(func() {
+			DeleteNamespaces(virtualCluster.Cluster.Namespace)
+		})
 
 		cluster := NewCluster(namespace.Name)
 		cluster.Spec.Persistence.Type = v1beta1.DynamicPersistenceMode
@@ -164,8 +165,6 @@ var _ = When("a dynamic cluster is installed", Label("e2e"), func() {
 
 		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
 
-		By(fmt.Sprintf("Created virtual cluster %s/%s", cluster.Namespace, cluster.Name))
-
 		virtualCluster := &VirtualCluster{
 			Cluster:    cluster,
 			RestConfig: restConfig,

tests/cluster_status_test.go

@@ -18,7 +18,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("a cluster's status is tracked", Label("e2e"), func() {
+var _ = When("a cluster's status is tracked", Label("e2e"), Label(statusTestsLabel), func() {
 	var (
 		namespace *corev1.Namespace
 		vcp       *v1beta1.VirtualClusterPolicy
@@ -27,7 +27,6 @@ var _ = When("a cluster's status is tracked", Label("e2e"), func() {
 	// This BeforeEach/AfterEach will create a new namespace and a default policy for each test.
 	BeforeEach(func() {
 		ctx := context.Background()
-		namespace = NewNamespace()
 
 		vcp = &v1beta1.VirtualClusterPolicy{
 			ObjectMeta: metav1.ObjectMeta{
@@ -36,6 +35,11 @@ var _ = When("a cluster's status is tracked", Label("e2e"), func() {
 		}
 		Expect(k8sClient.Create(ctx, vcp)).To(Succeed())
 
+		namespace = NewNamespace()
+
+		err := k8sClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace)
+		Expect(err).To(Not(HaveOccurred()))
+
 		namespace.Labels = map[string]string{
 			policy.PolicyNameLabelKey: vcp.Name,
 		}

tests/cluster_sync_test.go

@@ -0,0 +1,145 @@
+package k3k_test
+
+import (
+	"context"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/rancher/k3k/k3k-kubelet/translate"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = When("a shared mode cluster is created", Ordered, Label("e2e"), func() {
+	var (
+		virtualCluster   *VirtualCluster
+		virtualConfigMap *corev1.ConfigMap
+		virtualService   *corev1.Service
+	)
+
+	BeforeAll(func() {
+		virtualCluster = NewVirtualCluster()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(virtualCluster.Cluster.Namespace)
+		})
+	})
+
+	When("a ConfigMap is created in the virtual cluster", func() {
+		BeforeAll(func() {
+			ctx := context.Background()
+
+			virtualConfigMap = &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-cm",
+					Namespace: "default",
+				},
+			}
+
+			var err error
+
+			virtualConfigMap, err = virtualCluster.Client.CoreV1().ConfigMaps("default").Create(ctx, virtualConfigMap, metav1.CreateOptions{})
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("is replicated in the host cluster", func() {
+			ctx := context.Background()
+
+			hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster)
+			namespacedName := hostTranslator.NamespacedName(virtualConfigMap)
+
+			// check that the ConfigMap is synced in the host cluster
+			Eventually(func(g Gomega) {
+				_, err := k8s.CoreV1().ConfigMaps(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{})
+				g.Expect(err).To(Not(HaveOccurred()))
+			}).
+				WithTimeout(time.Minute).
+				WithPolling(time.Second).
+				Should(Succeed())
+		})
+	})
+
+	When("a Service is created in the virtual cluster", func() {
+		BeforeAll(func() {
+			ctx := context.Background()
+
+			virtualService = &corev1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-svc",
+					Namespace: "default",
+				},
+				Spec: corev1.ServiceSpec{
+					Type:  corev1.ServiceTypeClusterIP,
+					Ports: []corev1.ServicePort{{Port: 8888}},
+				},
+			}
+
+			var err error
+			virtualService, err = virtualCluster.Client.CoreV1().Services("default").Create(ctx, virtualService, metav1.CreateOptions{})
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("is replicated in the host cluster", func() {
+			ctx := context.Background()
+
+			hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster)
+			namespacedName := hostTranslator.NamespacedName(virtualService)
+
+			// check that the ConfigMap is synced in the host cluster
+			Eventually(func(g Gomega) {
+				_, err := k8s.CoreV1().Services(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{})
+				g.Expect(err).To(Not(HaveOccurred()))
+			}).
+				WithTimeout(time.Minute).
+				WithPolling(time.Second).
+				Should(Succeed())
+		})
+	})
+
+	When("the cluster is deleted", func() {
+		BeforeAll(func() {
+			ctx := context.Background()
+
+			By("Deleting cluster")
+
+			err := k8sClient.Delete(ctx, virtualCluster.Cluster)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("will delete the ConfigMap from the host cluster", func() {
+			ctx := context.Background()
+
+			hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster)
+			namespacedName := hostTranslator.NamespacedName(virtualConfigMap)
+
+			// check that the ConfigMap is deleted from the host cluster
+			Eventually(func(g Gomega) {
+				_, err := k8s.CoreV1().ConfigMaps(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{})
+				g.Expect(apierrors.IsNotFound(err)).To(BeTrue())
+			}).
+				WithTimeout(time.Minute).
+				WithPolling(time.Second).
+				Should(Succeed())
+		})
+
+		It("will delete the Service from the host cluster", func() {
+			ctx := context.Background()
+
+			hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster)
+			namespacedName := hostTranslator.NamespacedName(virtualService)
+
+			// check that the Service is deleted from the host cluster
+			Eventually(func(g Gomega) {
+				_, err := k8s.CoreV1().Services(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{})
+				g.Expect(apierrors.IsNotFound(err)).To(BeTrue())
+			}).
+				WithTimeout(time.Minute).
+				WithPolling(time.Second).
+				Should(Succeed())
+		})
+	})
+})

tests/cluster_update_test.go

@@ -18,12 +18,16 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("a shared mode cluster update its envs", Label("e2e"), func() {
+var _ = When("a shared mode cluster update its envs", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 	ctx := context.Background()
 	BeforeEach(func() {
 		namespace := NewNamespace()
 
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		cluster := NewCluster(namespace.Name)
 
 		// Add initial environment variables for server
@@ -71,6 +75,212 @@ var _ = When("a shared mode cluster update its envs", Label("e2e"), func() {
 		Expect(ok).To(BeTrue())
 		Expect(serverEnv2).To(Equal("toBeRemoved"))
 
+		var nodes v1.NodeList
+		Expect(k8sClient.List(ctx, &nodes)).To(Succeed())
+
+		aPods := listAgentPods(ctx, virtualCluster)
+		Expect(aPods).To(HaveLen(len(nodes.Items)))
+
+		agentPod := aPods[0]
+
+		agentEnv1, ok := getEnv(&agentPod, "TEST_AGENT_ENV_1")
+		Expect(ok).To(BeTrue())
+		Expect(agentEnv1).To(Equal("not_upgraded"))
+
+		agentEnv2, ok := getEnv(&agentPod, "TEST_AGENT_ENV_2")
+		Expect(ok).To(BeTrue())
+		Expect(agentEnv2).To(Equal("toBeRemoved"))
+	})
+	It("will update server and agent envs when cluster is updated", func() {
+		Eventually(func(g Gomega) {
+			var cluster v1beta1.Cluster
+
+			err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
+			g.Expect(err).NotTo(HaveOccurred())
+
+			// update both agent and server envs
+			cluster.Spec.ServerEnvs = []v1.EnvVar{
+				{
+					Name:  "TEST_SERVER_ENV_1",
+					Value: "upgraded",
+				},
+				{
+					Name:  "TEST_SERVER_ENV_3",
+					Value: "new",
+				},
+			}
+			cluster.Spec.AgentEnvs = []v1.EnvVar{
+				{
+					Name:  "TEST_AGENT_ENV_1",
+					Value: "upgraded",
+				},
+				{
+					Name:  "TEST_AGENT_ENV_3",
+					Value: "new",
+				},
+			}
+
+			err = k8sClient.Update(ctx, &cluster)
+			g.Expect(err).NotTo(HaveOccurred())
+
+			// server pods
+			serverPods := listServerPods(ctx, virtualCluster)
+			g.Expect(len(serverPods)).To(Equal(1))
+
+			serverEnv1, ok := getEnv(&serverPods[0], "TEST_SERVER_ENV_1")
+			g.Expect(ok).To(BeTrue())
+			g.Expect(serverEnv1).To(Equal("upgraded"))
+
+			_, ok = getEnv(&serverPods[0], "TEST_SERVER_ENV_2")
+			g.Expect(ok).To(BeFalse())
+
+			serverEnv3, ok := getEnv(&serverPods[0], "TEST_SERVER_ENV_3")
+			g.Expect(ok).To(BeTrue())
+			g.Expect(serverEnv3).To(Equal("new"))
+
+			// agent pods
+			var nodes v1.NodeList
+			g.Expect(k8sClient.List(ctx, &nodes)).To(Succeed())
+
+			aPods := listAgentPods(ctx, virtualCluster)
+			g.Expect(aPods).To(HaveLen(len(nodes.Items)))
+
+			agentEnv1, ok := getEnv(&aPods[0], "TEST_AGENT_ENV_1")
+			g.Expect(ok).To(BeTrue())
+			g.Expect(agentEnv1).To(Equal("upgraded"))
+
+			_, ok = getEnv(&aPods[0], "TEST_AGENT_ENV_2")
+			g.Expect(ok).To(BeFalse())
+
+			agentEnv3, ok := getEnv(&aPods[0], "TEST_AGENT_ENV_3")
+			g.Expect(ok).To(BeTrue())
+			g.Expect(agentEnv3).To(Equal("new"))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 2).
+			Should(Succeed())
+	})
+})
+
+var _ = When("a shared mode cluster update its server args", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+	var virtualCluster *VirtualCluster
+	ctx := context.Background()
+	BeforeEach(func() {
+		namespace := NewNamespace()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
+		cluster := NewCluster(namespace.Name)
+
+		// Add initial args for server
+		cluster.Spec.ServerArgs = []string{
+			"--node-label=test_server=not_upgraded",
+		}
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+		sPods := listServerPods(ctx, virtualCluster)
+		Expect(len(sPods)).To(Equal(1))
+
+		serverPod := sPods[0]
+
+		Expect(isArgFound(&serverPod, "--node-label=test_server=not_upgraded")).To(BeTrue())
+	})
+	It("will update server args", func() {
+		Eventually(func(g Gomega) {
+			var cluster v1beta1.Cluster
+
+			err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
+			g.Expect(err).NotTo(HaveOccurred())
+
+			cluster.Spec.ServerArgs = []string{
+				"--node-label=test_server=upgraded",
+			}
+
+			err = k8sClient.Update(ctx, &cluster)
+			g.Expect(err).NotTo(HaveOccurred())
+
+			// server pods
+			sPods := listServerPods(ctx, virtualCluster)
+			g.Expect(len(sPods)).To(Equal(1))
+
+			g.Expect(isArgFound(&sPods[0], "--node-label=test_server=upgraded")).To(BeTrue())
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 2).
+			Should(Succeed())
+	})
+})
+
+var _ = When("a virtual mode cluster update its envs", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+	var virtualCluster *VirtualCluster
+	ctx := context.Background()
+	BeforeEach(func() {
+		namespace := NewNamespace()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
+		cluster := NewCluster(namespace.Name)
+
+		// Add initial environment variables for server
+		cluster.Spec.ServerEnvs = []v1.EnvVar{
+			{
+				Name:  "TEST_SERVER_ENV_1",
+				Value: "not_upgraded",
+			},
+			{
+				Name:  "TEST_SERVER_ENV_2",
+				Value: "toBeRemoved",
+			},
+		}
+		// Add initial environment variables for agent
+		cluster.Spec.AgentEnvs = []v1.EnvVar{
+			{
+				Name:  "TEST_AGENT_ENV_1",
+				Value: "not_upgraded",
+			},
+			{
+				Name:  "TEST_AGENT_ENV_2",
+				Value: "toBeRemoved",
+			},
+		}
+
+		cluster.Spec.Mode = v1beta1.VirtualClusterMode
+		cluster.Spec.Agents = ptr.To[int32](1)
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+		sPods := listServerPods(ctx, virtualCluster)
+		Expect(len(sPods)).To(Equal(1))
+
+		serverPod := sPods[0]
+
+		serverEnv1, ok := getEnv(&serverPod, "TEST_SERVER_ENV_1")
+		Expect(ok).To(BeTrue())
+		Expect(serverEnv1).To(Equal("not_upgraded"))
+
+		serverEnv2, ok := getEnv(&serverPod, "TEST_SERVER_ENV_2")
+		Expect(ok).To(BeTrue())
+		Expect(serverEnv2).To(Equal("toBeRemoved"))
+
 		aPods := listAgentPods(ctx, virtualCluster)
 		Expect(len(aPods)).To(Equal(1))
 
@@ -152,12 +362,16 @@ var _ = When("a shared mode cluster update its envs", Label("e2e"), func() {
 	})
 })
 
-var _ = When("a shared mode cluster update its server args", Label("e2e"), func() {
+var _ = When("a virtual mode cluster update its server args", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 	ctx := context.Background()
 	BeforeEach(func() {
 		namespace := NewNamespace()
 
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		cluster := NewCluster(namespace.Name)
 
 		// Add initial args for server
@@ -165,6 +379,9 @@ var _ = When("a shared mode cluster update its server args", Label("e2e"), func(
 			"--node-label=test_server=not_upgraded",
 		}
 
+		cluster.Spec.Mode = v1beta1.VirtualClusterMode
+		cluster.Spec.Agents = ptr.To[int32](1)
+
 		CreateCluster(cluster)
 
 		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
@@ -207,202 +424,7 @@ var _ = When("a shared mode cluster update its server args", Label("e2e"), func(
 	})
 })
 
-var _ = When("a virtual mode cluster update its envs", Label("e2e"), func() {
-	var virtualCluster *VirtualCluster
-	ctx := context.Background()
-	BeforeEach(func() {
-		namespace := NewNamespace()
-
-		cluster := NewCluster(namespace.Name)
-
-		// Add initial environment variables for server
-		cluster.Spec.ServerEnvs = []v1.EnvVar{
-			{
-				Name:  "TEST_SERVER_ENV_1",
-				Value: "not_upgraded",
-			},
-			{
-				Name:  "TEST_SERVER_ENV_2",
-				Value: "toBeRemoved",
-			},
-		}
-		// Add initial environment variables for agent
-		cluster.Spec.AgentEnvs = []v1.EnvVar{
-			{
-				Name:  "TEST_AGENT_ENV_1",
-				Value: "not_upgraded",
-			},
-			{
-				Name:  "TEST_AGENT_ENV_2",
-				Value: "toBeRemoved",
-			},
-		}
-
-		cluster.Spec.Mode = v1beta1.VirtualClusterMode
-		cluster.Spec.Agents = ptr.To(int32(1))
-
-		CreateCluster(cluster)
-
-		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
-
-		virtualCluster = &VirtualCluster{
-			Cluster:    cluster,
-			RestConfig: restConfig,
-			Client:     client,
-		}
-		sPods := listServerPods(ctx, virtualCluster)
-		Expect(len(sPods)).To(Equal(1))
-
-		serverPod := sPods[0]
-
-		serverEnv1, ok := getEnv(&serverPod, "TEST_SERVER_ENV_1")
-		Expect(ok).To(BeTrue())
-		Expect(serverEnv1).To(Equal("not_upgraded"))
-
-		serverEnv2, ok := getEnv(&serverPod, "TEST_SERVER_ENV_2")
-		Expect(ok).To(BeTrue())
-		Expect(serverEnv2).To(Equal("toBeRemoved"))
-
-		aPods := listAgentPods(ctx, virtualCluster)
-		Expect(len(aPods)).To(Equal(1))
-
-		agentPod := aPods[0]
-
-		agentEnv1, ok := getEnv(&agentPod, "TEST_AGENT_ENV_1")
-		Expect(ok).To(BeTrue())
-		Expect(agentEnv1).To(Equal("not_upgraded"))
-
-		agentEnv2, ok := getEnv(&agentPod, "TEST_AGENT_ENV_2")
-		Expect(ok).To(BeTrue())
-		Expect(agentEnv2).To(Equal("toBeRemoved"))
-	})
-	It("will update server and agent envs when cluster is updated", func() {
-		Eventually(func(g Gomega) {
-			var cluster v1beta1.Cluster
-
-			err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
-			g.Expect(err).NotTo(HaveOccurred())
-
-			// update both agent and server envs
-			cluster.Spec.ServerEnvs = []v1.EnvVar{
-				{
-					Name:  "TEST_SERVER_ENV_1",
-					Value: "upgraded",
-				},
-				{
-					Name:  "TEST_SERVER_ENV_3",
-					Value: "new",
-				},
-			}
-			cluster.Spec.AgentEnvs = []v1.EnvVar{
-				{
-					Name:  "TEST_AGENT_ENV_1",
-					Value: "upgraded",
-				},
-				{
-					Name:  "TEST_AGENT_ENV_3",
-					Value: "new",
-				},
-			}
-
-			err = k8sClient.Update(ctx, &cluster)
-			g.Expect(err).NotTo(HaveOccurred())
-
-			// server pods
-			serverPods := listServerPods(ctx, virtualCluster)
-			g.Expect(len(serverPods)).To(Equal(1))
-
-			serverEnv1, ok := getEnv(&serverPods[0], "TEST_SERVER_ENV_1")
-			g.Expect(ok).To(BeTrue())
-			g.Expect(serverEnv1).To(Equal("upgraded"))
-
-			_, ok = getEnv(&serverPods[0], "TEST_SERVER_ENV_2")
-			g.Expect(ok).To(BeFalse())
-
-			serverEnv3, ok := getEnv(&serverPods[0], "TEST_SERVER_ENV_3")
-			g.Expect(ok).To(BeTrue())
-			g.Expect(serverEnv3).To(Equal("new"))
-
-			// agent pods
-			aPods := listAgentPods(ctx, virtualCluster)
-			g.Expect(len(aPods)).To(Equal(1))
-
-			agentEnv1, ok := getEnv(&aPods[0], "TEST_AGENT_ENV_1")
-			g.Expect(ok).To(BeTrue())
-			g.Expect(agentEnv1).To(Equal("upgraded"))
-
-			_, ok = getEnv(&aPods[0], "TEST_AGENT_ENV_2")
-			g.Expect(ok).To(BeFalse())
-
-			agentEnv3, ok := getEnv(&aPods[0], "TEST_AGENT_ENV_3")
-			g.Expect(ok).To(BeTrue())
-			g.Expect(agentEnv3).To(Equal("new"))
-		}).
-			WithPolling(time.Second * 2).
-			WithTimeout(time.Minute * 2).
-			Should(Succeed())
-	})
-})
-
-var _ = When("a virtual mode cluster update its server args", Label("e2e"), func() {
-	var virtualCluster *VirtualCluster
-	ctx := context.Background()
-	BeforeEach(func() {
-		namespace := NewNamespace()
-
-		cluster := NewCluster(namespace.Name)
-
-		// Add initial args for server
-		cluster.Spec.ServerArgs = []string{
-			"--node-label=test_server=not_upgraded",
-		}
-
-		cluster.Spec.Mode = v1beta1.VirtualClusterMode
-		cluster.Spec.Agents = ptr.To(int32(1))
-
-		CreateCluster(cluster)
-
-		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
-
-		virtualCluster = &VirtualCluster{
-			Cluster:    cluster,
-			RestConfig: restConfig,
-			Client:     client,
-		}
-		sPods := listServerPods(ctx, virtualCluster)
-		Expect(len(sPods)).To(Equal(1))
-
-		serverPod := sPods[0]
-
-		Expect(isArgFound(&serverPod, "--node-label=test_server=not_upgraded")).To(BeTrue())
-	})
-	It("will update server args", func() {
-		Eventually(func(g Gomega) {
-			var cluster v1beta1.Cluster
-
-			err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
-			g.Expect(err).NotTo(HaveOccurred())
-
-			cluster.Spec.ServerArgs = []string{
-				"--node-label=test_server=upgraded",
-			}
-
-			err = k8sClient.Update(ctx, &cluster)
-			g.Expect(err).NotTo(HaveOccurred())
-
-			// server pods
-			sPods := listServerPods(ctx, virtualCluster)
-			g.Expect(len(sPods)).To(Equal(1))
-
-			g.Expect(isArgFound(&sPods[0], "--node-label=test_server=upgraded")).To(BeTrue())
-		}).
-			WithPolling(time.Second * 2).
-			WithTimeout(time.Minute * 2).
-			Should(Succeed())
-	})
-})
-
-var _ = When("a shared mode cluster update its version", Label("e2e"), func() {
+var _ = When("a shared mode cluster update its version", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		virtualCluster *VirtualCluster
 		nginxPod       *v1.Pod
@@ -411,6 +433,10 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() {
 		ctx := context.Background()
 		namespace := NewNamespace()
 
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		cluster := NewCluster(namespace.Name)
 
 		// Add initial version
@@ -437,7 +463,12 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() {
 		Expect(serverPod.Spec.Containers[0].Image).To(Equal("rancher/k3s:" + cluster.Spec.Version))
 
 		nginxPod, _ = virtualCluster.NewNginxPod("")
+
+		DeferCleanup(func() {
+			DeleteNamespaces(virtualCluster.Cluster.Namespace)
+		})
 	})
+
 	It("will update server version when version spec is updated", func() {
 		var cluster v1beta1.Cluster
 		ctx := context.Background()
@@ -457,8 +488,7 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() {
 			g.Expect(len(serverPods)).To(Equal(1))
 
 			serverPod := serverPods[0]
-			condIndex, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
-			g.Expect(condIndex).NotTo(Equal(-1))
+			_, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
 			g.Expect(cond).NotTo(BeNil())
 			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
 
@@ -468,20 +498,19 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() {
 			g.Expect(err).To(BeNil())
 			g.Expect(clusterVersion.String()).To(Equal(strings.ReplaceAll(cluster.Spec.Version, "-", "+")))
 
-			_, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
+			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
 			g.Expect(err).To(BeNil())
-			condIndex, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
-			g.Expect(condIndex).NotTo(Equal(-1))
+			_, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
 			g.Expect(cond).NotTo(BeNil())
 			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
 		}).
-			WithPolling(time.Second * 2).
+			WithPolling(time.Second * 5).
 			WithTimeout(time.Minute * 3).
 			Should(Succeed())
 	})
 })
 
-var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
+var _ = When("a virtual mode cluster update its version", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		virtualCluster *VirtualCluster
 		nginxPod       *v1.Pod
@@ -490,13 +519,17 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
 		ctx := context.Background()
 		namespace := NewNamespace()
 
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		cluster := NewCluster(namespace.Name)
 
 		// Add initial version
 		cluster.Spec.Version = "v1.31.13-k3s1"
 
 		cluster.Spec.Mode = v1beta1.VirtualClusterMode
-		cluster.Spec.Agents = ptr.To(int32(1))
+		cluster.Spec.Agents = ptr.To[int32](1)
 
 		// need to enable persistence for this
 		cluster.Spec.Persistence = v1beta1.PersistenceConfig{
@@ -544,8 +577,7 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
 			g.Expect(len(serverPods)).To(Equal(1))
 
 			serverPod := serverPods[0]
-			condIndex, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
-			g.Expect(condIndex).NotTo(Equal(-1))
+			_, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
 			g.Expect(cond).NotTo(BeNil())
 			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
 
@@ -556,8 +588,7 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
 			g.Expect(len(agentPods)).To(Equal(1))
 
 			agentPod := agentPods[0]
-			condIndex, cond = pod.GetPodCondition(&agentPod.Status, v1.PodReady)
-			g.Expect(condIndex).NotTo(Equal(-1))
+			_, cond = pod.GetPodCondition(&agentPod.Status, v1.PodReady)
 			g.Expect(cond).NotTo(BeNil())
 			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
 
@@ -570,8 +601,7 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
 			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
 			g.Expect(err).To(BeNil())
 
-			condIndex, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
-			g.Expect(condIndex).NotTo(Equal(-1))
+			_, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
 			g.Expect(cond).NotTo(BeNil())
 			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
 		}).
@@ -580,3 +610,381 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
 			Should(Succeed())
 	})
 })
+
+var _ = When("a shared mode cluster scales up servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+	var (
+		virtualCluster *VirtualCluster
+		nginxPod       *v1.Pod
+	)
+	BeforeEach(func() {
+		ctx := context.Background()
+		namespace := NewNamespace()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
+		cluster := NewCluster(namespace.Name)
+
+		// need to enable persistence for this
+		cluster.Spec.Persistence = v1beta1.PersistenceConfig{
+			Type: v1beta1.DynamicPersistenceMode,
+		}
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+		sPods := listServerPods(ctx, virtualCluster)
+		Expect(len(sPods)).To(Equal(1))
+
+		Eventually(func(g Gomega) {
+			// since there is no way to check nodes in shared mode
+			// we can check if the endpoints are registered to N nodes
+			k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(1))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 3).
+			Should(Succeed())
+
+		nginxPod, _ = virtualCluster.NewNginxPod("")
+	})
+	It("will scale up server pods", func() {
+		var cluster v1beta1.Cluster
+		ctx := context.Background()
+
+		err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		// scale cluster servers to 3 nodes
+		cluster.Spec.Servers = ptr.To[int32](3)
+
+		err = k8sClient.Update(ctx, &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func(g Gomega) {
+			// server pods
+			serverPods := listServerPods(ctx, virtualCluster)
+			g.Expect(len(serverPods)).To(Equal(3))
+
+			for _, serverPod := range serverPods {
+				_, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
+				g.Expect(cond).NotTo(BeNil())
+				g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+			}
+
+			k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(3))
+
+			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
+			g.Expect(err).To(BeNil())
+			_, cond := pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 3).
+			Should(Succeed())
+	})
+})
+
+var _ = When("a shared mode cluster scales down servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+	var (
+		virtualCluster *VirtualCluster
+		nginxPod       *v1.Pod
+	)
+	BeforeEach(func() {
+		ctx := context.Background()
+		namespace := NewNamespace()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
+		cluster := NewCluster(namespace.Name)
+
+		// start cluster with 3 servers
+		cluster.Spec.Servers = ptr.To[int32](3)
+
+		// need to enable persistence for this
+		cluster.Spec.Persistence = v1beta1.PersistenceConfig{
+			Type: v1beta1.DynamicPersistenceMode,
+		}
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+		// no need to check servers status since createCluster() will wait until all servers are in ready state
+		sPods := listServerPods(ctx, virtualCluster)
+		Expect(len(sPods)).To(Equal(3))
+
+		Eventually(func(g Gomega) {
+			// since there is no way to check nodes in shared mode
+			// we can check if the endpoints are registered to N nodes
+			k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(3))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 3).
+			Should(Succeed())
+
+		nginxPod, _ = virtualCluster.NewNginxPod("")
+	})
+	It("will scale down server pods", func() {
+		var cluster v1beta1.Cluster
+		ctx := context.Background()
+
+		err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		// scale down cluster servers to 1 node
+		cluster.Spec.Servers = ptr.To[int32](1)
+
+		err = k8sClient.Update(ctx, &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func(g Gomega) {
+			// server pods
+			serverPods := listServerPods(ctx, virtualCluster)
+			g.Expect(len(serverPods)).To(Equal(1))
+
+			_, cond := pod.GetPodCondition(&serverPods[0].Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+
+			k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(1))
+
+			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
+			g.Expect(err).To(BeNil())
+			_, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 3).
+			Should(Succeed())
+	})
+})
+
+var _ = When("a virtual mode cluster scales up servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+	var (
+		virtualCluster *VirtualCluster
+		nginxPod       *v1.Pod
+	)
+	BeforeEach(func() {
+		ctx := context.Background()
+		namespace := NewNamespace()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
+		cluster := NewCluster(namespace.Name)
+
+		cluster.Spec.Mode = v1beta1.VirtualClusterMode
+
+		// need to enable persistence for this
+		cluster.Spec.Persistence = v1beta1.PersistenceConfig{
+			Type: v1beta1.DynamicPersistenceMode,
+		}
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+		sPods := listServerPods(ctx, virtualCluster)
+		Expect(len(sPods)).To(Equal(1))
+
+		Eventually(func(g Gomega) {
+			nodes, err := virtualCluster.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(len(nodes.Items)).To(Equal(1))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 5).
+			Should(Succeed())
+
+		nginxPod, _ = virtualCluster.NewNginxPod("")
+	})
+	It("will scale up server pods", func() {
+		var cluster v1beta1.Cluster
+		ctx := context.Background()
+
+		err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		// scale cluster servers to 3 nodes
+		cluster.Spec.Servers = ptr.To[int32](3)
+
+		err = k8sClient.Update(ctx, &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func(g Gomega) {
+			// server pods
+			serverPods := listServerPods(ctx, virtualCluster)
+			g.Expect(len(serverPods)).To(Equal(3))
+
+			for _, serverPod := range serverPods {
+				_, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
+				g.Expect(cond).NotTo(BeNil())
+				g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+			}
+
+			nodes, err := virtualCluster.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(len(nodes.Items)).To(Equal(3))
+
+			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
+			g.Expect(err).To(BeNil())
+			_, cond := pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 5).
+			Should(Succeed())
+	})
+})
+
+var _ = When("a virtual mode cluster scales down servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+	var (
+		virtualCluster *VirtualCluster
+		nginxPod       *v1.Pod
+	)
+	BeforeEach(func() {
+		ctx := context.Background()
+		namespace := NewNamespace()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
+		cluster := NewCluster(namespace.Name)
+
+		cluster.Spec.Mode = v1beta1.VirtualClusterMode
+
+		// start cluster with 3 servers
+		cluster.Spec.Servers = ptr.To[int32](3)
+
+		// need to enable persistence for this
+		cluster.Spec.Persistence = v1beta1.PersistenceConfig{
+			Type: v1beta1.DynamicPersistenceMode,
+		}
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+		// no need to check servers status since createCluster() will wait until all servers are in ready state
+		sPods := listServerPods(ctx, virtualCluster)
+		Expect(len(sPods)).To(Equal(3))
+
+		Eventually(func(g Gomega) {
+			nodes, err := virtualCluster.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(len(nodes.Items)).To(Equal(3))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 5).
+			Should(Succeed())
+
+		nginxPod, _ = virtualCluster.NewNginxPod("")
+	})
+
+	It("will scale down server pods", func() {
+		By("Scaling down cluster")
+
+		var cluster v1beta1.Cluster
+		ctx := context.Background()
+
+		err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		// scale down cluster servers to 1 node
+		cluster.Spec.Servers = ptr.To[int32](1)
+
+		err = k8sClient.Update(ctx, &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func(g Gomega) {
+			serverPods := listServerPods(ctx, virtualCluster)
+
+			// Wait for all the server pods to be marked for deletion
+			for _, serverPod := range serverPods {
+				g.Expect(serverPod.DeletionTimestamp).NotTo(BeNil())
+			}
+		}).
+			MustPassRepeatedly(5).
+			WithPolling(time.Second * 5).
+			WithTimeout(time.Minute * 3).
+			Should(Succeed())
+
+		By("Waiting for cluster to be ready again")
+
+		Eventually(func(g Gomega) {
+			// server pods
+			serverPods := listServerPods(ctx, virtualCluster)
+			g.Expect(len(serverPods)).To(Equal(1))
+
+			_, cond := pod.GetPodCondition(&serverPods[0].Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+
+			// we can't check for number of nodes in scale down because the nodes will be there but in a non-ready state
+			k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(1))
+		}).
+			MustPassRepeatedly(5).
+			WithPolling(time.Second * 5).
+			WithTimeout(time.Minute * 2).
+			Should(Succeed())
+
+		By("Checking that Nginx Pod is Running")
+
+		Eventually(func(g Gomega) {
+			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
+			g.Expect(err).To(BeNil())
+
+			// TODO: there is a possible issue where the Pod is not being marked as Ready
+			// if the kubelet lost the sync with the API server.
+			// We check for the ContainersReady status (all containers in the pod are ready),
+			// but this is probably to investigate.
+			// Related issue (?): https://github.com/kubernetes/kubernetes/issues/82346
+
+			_, cond := pod.GetPodCondition(&nginxPod.Status, v1.ContainersReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute).
+			Should(Succeed())
+	})
+})

tests/common_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -14,10 +15,11 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/kubectl/pkg/scheme"
+	"k8s.io/kubernetes/pkg/api/v1/pod"
 	"k8s.io/utils/ptr"
 
-	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 
 	"github.com/rancher/k3k/k3k-kubelet/translate"
@@ -46,8 +48,6 @@ func NewVirtualClusterWithType(persistenceType v1beta1.PersistenceMode) *Virtual
 
 	namespace := NewNamespace()
 
-	By(fmt.Sprintf("Creating new virtual cluster in namespace %s", namespace.Name))
-
 	cluster := NewCluster(namespace.Name)
 	cluster.Spec.Persistence.Type = persistenceType
 
@@ -87,11 +87,11 @@ func NewVirtualClusters(n int) []*VirtualCluster {
 	return clusters
 }
 
-func NewNamespace() *corev1.Namespace {
+func NewNamespace() *v1.Namespace {
 	GinkgoHelper()
 
-	namespace := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-", Labels: map[string]string{"e2e": "true"}}}
-	namespace, err := k8s.CoreV1().Namespaces().Create(context.Background(), namespace, v1.CreateOptions{})
+	namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-", Labels: map[string]string{"e2e": "true"}}}
+	namespace, err := k8s.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
 	Expect(err).To(Not(HaveOccurred()))
 
 	return namespace
@@ -100,6 +100,11 @@ func NewNamespace() *corev1.Namespace {
 func DeleteNamespaces(names ...string) {
 	GinkgoHelper()
 
+	if _, found := os.LookupEnv("KEEP_NAMESPACES"); found {
+		By(fmt.Sprintf("Keeping namespace %v", names))
+		return
+	}
+
 	wg := sync.WaitGroup{}
 	wg.Add(len(names))
 
@@ -120,7 +125,7 @@ func deleteNamespace(name string) {
 
 	By(fmt.Sprintf("Deleting namespace %s", name))
 
-	err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, v1.DeleteOptions{
+	err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{
 		GracePeriodSeconds: ptr.To[int64](0),
 	})
 	Expect(err).To(Not(HaveOccurred()))
@@ -128,7 +133,7 @@ func deleteNamespace(name string) {
 
 func NewCluster(namespace string) *v1beta1.Cluster {
 	return &v1beta1.Cluster{
-		ObjectMeta: v1.ObjectMeta{
+		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "cluster-",
 			Namespace:    namespace,
 		},
@@ -150,40 +155,53 @@ func NewCluster(namespace string) *v1beta1.Cluster {
 func CreateCluster(cluster *v1beta1.Cluster) {
 	GinkgoHelper()
 
+	By(fmt.Sprintf("Creating new virtual cluster in namespace %s", cluster.Namespace))
+
 	ctx := context.Background()
 	err := k8sClient.Create(ctx, cluster)
 	Expect(err).To(Not(HaveOccurred()))
 
-	By("Waiting for cluster to be ready")
+	expectedServers := int(*cluster.Spec.Servers)
+	expectedAgents := int(*cluster.Spec.Agents)
+
+	By(fmt.Sprintf("Waiting for cluster %s to be ready in namespace %s. Expected servers: %d. Expected agents: %d", cluster.Name, cluster.Namespace, expectedServers, expectedAgents))
+
+	// track the Eventually status to log for changes
+	prev := -1
 
 	// check that the server Pod and the Kubelet are in Ready state
 	Eventually(func() bool {
-		podList, err := k8s.CoreV1().Pods(cluster.Namespace).List(ctx, v1.ListOptions{})
+		podList, err := k8s.CoreV1().Pods(cluster.Namespace).List(ctx, metav1.ListOptions{})
 		Expect(err).To(Not(HaveOccurred()))
 
 		// all the servers and agents needs to be in a running phase
 		var serversReady, agentsReady int
 
-		for _, pod := range podList.Items {
-			if pod.Labels["role"] == "server" {
-				GinkgoLogr.Info(fmt.Sprintf("server pod=%s/%s status=%s", pod.Namespace, pod.Name, pod.Status.Phase))
-				if pod.Status.Phase == corev1.PodRunning {
-					serversReady++
-				}
+		for _, k3sPod := range podList.Items {
+			_, cond := pod.GetPodCondition(&k3sPod.Status, v1.PodReady)
+
+			// pod not ready
+			if cond == nil || cond.Status != v1.ConditionTrue {
+				continue
 			}
 
-			if pod.Labels["type"] == "agent" {
-				GinkgoLogr.Info(fmt.Sprintf("agent pod=%s/%s status=%s", pod.Namespace, pod.Name, pod.Status.Phase))
-				if pod.Status.Phase == corev1.PodRunning {
-					agentsReady++
-				}
+			if k3sPod.Labels["role"] == "server" {
+				serversReady++
+			}
+
+			if k3sPod.Labels["type"] == "agent" {
+				agentsReady++
 			}
 		}
 
-		expectedServers := int(*cluster.Spec.Servers)
-		expectedAgents := int(*cluster.Spec.Agents)
-
-		By(fmt.Sprintf("serversReady=%d/%d agentsReady=%d/%d", serversReady, expectedServers, agentsReady, expectedAgents))
+		if prev != (serversReady + agentsReady) {
+			GinkgoLogr.Info("Waiting for pods to be Ready",
+				"servers", serversReady, "agents", agentsReady,
+				"name", cluster.Name, "namespace", cluster.Namespace,
+				"time", time.Now().Format(time.DateTime),
+			)
+			prev = (serversReady + agentsReady)
+		}
 
 		// the server pods should equal the expected servers, but since in shared mode we also have the kubelet is fine to have more than one
 		if (serversReady != expectedServers) || (agentsReady < expectedAgents) {
@@ -193,8 +211,10 @@ func CreateCluster(cluster *v1beta1.Cluster) {
 		return true
 	}).
 		WithTimeout(time.Minute * 5).
-		WithPolling(time.Second * 5).
+		WithPolling(time.Second * 10).
 		Should(BeTrue())
+
+	By("Cluster is ready")
 }
 
 // NewVirtualK8sClient returns a Kubernetes ClientSet for the virtual cluster
@@ -236,41 +256,60 @@ func NewVirtualK8sClientAndConfig(cluster *v1beta1.Cluster) (*kubernetes.Clients
 	return virtualK8sClient, restcfg
 }
 
-func (c *VirtualCluster) NewNginxPod(namespace string) (*corev1.Pod, string) {
+func (c *VirtualCluster) NewNginxPod(namespace string) (*v1.Pod, string) {
 	GinkgoHelper()
 
 	if namespace == "" {
 		namespace = "default"
 	}
 
-	nginxPod := &corev1.Pod{
-		ObjectMeta: v1.ObjectMeta{
+	nginxPod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "nginx-",
 			Namespace:    namespace,
 		},
-		Spec: corev1.PodSpec{
-			Containers: []corev1.Container{{
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{{
 				Name:  "nginx",
 				Image: "nginx",
 			}},
 		},
 	}
 
-	By("Creating Pod")
+	By("Creating Nginx Pod and waiting for it to be Ready")
 
 	ctx := context.Background()
-	nginxPod, err := c.Client.CoreV1().Pods(nginxPod.Namespace).Create(ctx, nginxPod, v1.CreateOptions{})
+
+	var err error
+
+	nginxPod, err = c.Client.CoreV1().Pods(nginxPod.Namespace).Create(ctx, nginxPod, metav1.CreateOptions{})
 	Expect(err).To(Not(HaveOccurred()))
 
-	var podIP string
+	// check that the nginx Pod is up and running in the virtual cluster
+	Eventually(func(g Gomega) {
+		nginxPod, err = c.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
+		g.Expect(err).To(Not(HaveOccurred()))
+
+		_, cond := pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
+		g.Expect(cond).NotTo(BeNil())
+		g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+	}).
+		WithTimeout(time.Minute).
+		WithPolling(time.Second).
+		Should(Succeed())
+
+	By(fmt.Sprintf("Nginx Pod is running (%s/%s)", nginxPod.Namespace, nginxPod.Name))
 
 	// only check the pod on the host cluster if the mode is shared mode
 	if c.Cluster.Spec.Mode != v1beta1.SharedClusterMode {
 		return nginxPod, ""
 	}
+
+	var podIP string
+
 	// check that the nginx Pod is up and running in the host cluster
 	Eventually(func() bool {
-		podList, err := k8s.CoreV1().Pods(c.Cluster.Namespace).List(ctx, v1.ListOptions{})
+		podList, err := k8s.CoreV1().Pods(c.Cluster.Namespace).List(ctx, metav1.ListOptions{})
 		Expect(err).To(Not(HaveOccurred()))
 
 		for _, pod := range podList.Items {
@@ -285,7 +324,7 @@ func (c *VirtualCluster) NewNginxPod(namespace string) (*corev1.Pod, string) {
 					pod.Name, resourceNamespace, resourceName, pod.Status.Phase, podIP,
 				)
 
-				return pod.Status.Phase == corev1.PodRunning && podIP != ""
+				return pod.Status.Phase == v1.PodRunning && podIP != ""
 			}
 		}
 
@@ -295,16 +334,12 @@ func (c *VirtualCluster) NewNginxPod(namespace string) (*corev1.Pod, string) {
 		WithPolling(time.Second * 5).
 		Should(BeTrue())
 
-	// get the running pod from the virtual cluster
-	nginxPod, err = c.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, v1.GetOptions{})
-	Expect(err).To(Not(HaveOccurred()))
-
 	return nginxPod, podIP
 }
 
 // ExecCmd exec command on specific pod and wait the command's output.
-func (c *VirtualCluster) ExecCmd(pod *corev1.Pod, command string) (string, string, error) {
-	option := &corev1.PodExecOptions{
+func (c *VirtualCluster) ExecCmd(pod *v1.Pod, command string) (string, string, error) {
+	option := &v1.PodExecOptions{
 		Command: []string{"sh", "-c", command},
 		Stdout:  true,
 		Stderr:  true,
@@ -336,7 +371,7 @@ func restartServerPod(ctx context.Context, virtualCluster *VirtualCluster) {
 	GinkgoHelper()
 
 	labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
-	serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+	serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
 	Expect(err).To(Not(HaveOccurred()))
 
 	Expect(len(serverPods.Items)).To(Equal(1))
@@ -344,40 +379,40 @@ func restartServerPod(ctx context.Context, virtualCluster *VirtualCluster) {
 
 	GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
 
-	err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{})
+	err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, metav1.DeleteOptions{})
 	Expect(err).To(Not(HaveOccurred()))
 
 	By("Deleting server pod")
 
 	// check that the server pods restarted
 	Eventually(func() any {
-		serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+		serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
 		Expect(err).To(Not(HaveOccurred()))
 		Expect(len(serverPods.Items)).To(Equal(1))
 		return serverPods.Items[0].DeletionTimestamp
 	}).WithTimeout(60 * time.Second).WithPolling(time.Second * 5).Should(BeNil())
 }
 
-func listServerPods(ctx context.Context, virtualCluster *VirtualCluster) []corev1.Pod {
+func listServerPods(ctx context.Context, virtualCluster *VirtualCluster) []v1.Pod {
 	labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
 
-	serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+	serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
 	Expect(err).To(Not(HaveOccurred()))
 
 	return serverPods.Items
 }
 
-func listAgentPods(ctx context.Context, virtualCluster *VirtualCluster) []corev1.Pod {
+func listAgentPods(ctx context.Context, virtualCluster *VirtualCluster) []v1.Pod {
 	labelSelector := fmt.Sprintf("cluster=%s,type=agent,mode=%s", virtualCluster.Cluster.Name, virtualCluster.Cluster.Spec.Mode)
 
-	agentPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+	agentPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
 	Expect(err).To(Not(HaveOccurred()))
 
 	return agentPods.Items
 }
 
 // getEnv will get an environment variable from a pod it will return empty string if not found
-func getEnv(pod *corev1.Pod, envName string) (string, bool) {
+func getEnv(pod *v1.Pod, envName string) (string, bool) {
 	container := pod.Spec.Containers[0]
 	for _, envVar := range container.Env {
 		if envVar.Name == envName {
@@ -389,7 +424,7 @@ func getEnv(pod *corev1.Pod, envName string) (string, bool) {
 }
 
 // isArgFound will return true if the argument passed to the function is found in container args
-func isArgFound(pod *corev1.Pod, arg string) bool {
+func isArgFound(pod *v1.Pod, arg string) bool {
 	container := pod.Spec.Containers[0]
 	for _, cmd := range container.Command {
 		if strings.Contains(cmd, arg) {

tests/installation_test.go

@@ -1,35 +0,0 @@
-package k3k_test
-
-import (
-	"context"
-	"time"
-
-	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-var _ = When("k3k is installed", Label("e2e"), func() {
-	It("is in Running status", func() {
-		// check that the controller is running
-		Eventually(func() bool {
-			opts := v1.ListOptions{LabelSelector: "app.kubernetes.io/name=k3k"}
-			podList, err := k8s.CoreV1().Pods(k3kNamespace).List(context.Background(), opts)
-
-			Expect(err).To(Not(HaveOccurred()))
-			Expect(podList.Items).To(Not(BeEmpty()))
-
-			for _, pod := range podList.Items {
-				if pod.Status.Phase == corev1.PodRunning {
-					return true
-				}
-			}
-			return false
-		}).
-			WithTimeout(time.Second * 10).
-			WithPolling(time.Second).
-			Should(BeTrue())
-	})
-})

tests/tests_suite_test.go

@@ -26,10 +26,11 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/kubernetes/pkg/api/v1/pod"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
-	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
@@ -40,7 +41,13 @@ import (
 
 const (
 	k3kNamespace = "k3k-system"
-	k3kName      = "k3k"
+
+	slowTestsLabel         = "slow"
+	updateTestsLabel       = "update"
+	persistenceTestsLabel  = "persistence"
+	networkingTestsLabel   = "networking"
+	statusTestsLabel       = "status"
+	certificatesTestsLabel = "certificates"
 )
 
 func TestTests(t *testing.T) {
@@ -116,7 +123,7 @@ func initKubernetesClient(ctx context.Context) {
 func buildScheme() *runtime.Scheme {
 	scheme := runtime.NewScheme()
 
-	err := corev1.AddToScheme(scheme)
+	err := v1.AddToScheme(scheme)
 	Expect(err).NotTo(HaveOccurred())
 	err = v1beta1.AddToScheme(scheme)
 	Expect(err).NotTo(HaveOccurred())
@@ -186,7 +193,7 @@ func installK3kChart() {
 	Expect(err).To(Not(HaveOccurred()))
 
 	iCli := action.NewInstall(helmActionConfig)
-	iCli.ReleaseName = k3kName
+	iCli.ReleaseName = "k3k"
 	iCli.Namespace = k3kNamespace
 	iCli.CreateNamespace = true
 	iCli.Timeout = time.Minute
@@ -215,76 +222,69 @@ func installK3kChart() {
 }
 
 func patchPVC(ctx context.Context, clientset *kubernetes.Clientset) {
-	pvc := &corev1.PersistentVolumeClaim{
+	deployments, err := clientset.AppsV1().Deployments(k3kNamespace).List(ctx, metav1.ListOptions{})
+	Expect(err).To(Not(HaveOccurred()))
+	Expect(deployments.Items).To(HaveLen(1))
+
+	k3kDeployment := &deployments.Items[0]
+
+	pvc := &v1.PersistentVolumeClaim{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "coverage-data-pvc",
 			Namespace: k3kNamespace,
 		},
-		Spec: corev1.PersistentVolumeClaimSpec{
-			AccessModes: []corev1.PersistentVolumeAccessMode{
-				corev1.ReadWriteOnce,
+		Spec: v1.PersistentVolumeClaimSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				v1.ReadWriteOnce,
 			},
-			Resources: corev1.VolumeResourceRequirements{
-				Requests: corev1.ResourceList{
-					corev1.ResourceStorage: resource.MustParse("100M"),
+			Resources: v1.VolumeResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceStorage: resource.MustParse("100M"),
 				},
 			},
 		},
 	}
 
-	_, err := clientset.CoreV1().PersistentVolumeClaims(k3kNamespace).Create(ctx, pvc, metav1.CreateOptions{})
-	Expect(err).To(Not(HaveOccurred()))
+	_, err = clientset.CoreV1().PersistentVolumeClaims(k3kNamespace).Create(ctx, pvc, metav1.CreateOptions{})
+	Expect(client.IgnoreAlreadyExists(err)).To(Not(HaveOccurred()))
 
-	patchData := []byte(`
-{
-    "spec": {
-        "template": {
-            "spec": {
-                "volumes": [
-                    {
-                        "name": "tmp-covdata",
-                        "persistentVolumeClaim": {
-                            "claimName": "coverage-data-pvc"
-                        }
-                    }
-                ],
-                "containers": [
-                    {
-                        "name": "k3k",
-                        "volumeMounts": [
-                            {
-                                "name": "tmp-covdata",
-                                "mountPath": "/tmp/covdata"
-                            }
-                        ],
-                        "env": [
-                            {
-                                "name": "GOCOVERDIR",
-                                "value": "/tmp/covdata"
-                            }
-                        ]
-                    }
-                ]
-            }
-        }
-    }
-}`)
+	k3kSpec := k3kDeployment.Spec.Template.Spec
 
-	GinkgoWriter.Printf("Applying patch to deployment '%s' in namespace '%s'...\n", k3kName, k3kNamespace)
+	// check if the Deployment has already the volume for the coverage
+	for _, volumes := range k3kSpec.Volumes {
+		if volumes.Name == "tmp-covdata" {
+			return
+		}
+	}
 
-	_, err = clientset.AppsV1().Deployments(k3kNamespace).Patch(
-		ctx,
-		k3kName,
-		types.StrategicMergePatchType,
-		patchData,
-		metav1.PatchOptions{},
-	)
+	k3kSpec.Volumes = append(k3kSpec.Volumes, v1.Volume{
+		Name: "tmp-covdata",
+		VolumeSource: v1.VolumeSource{
+			PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+				ClaimName: "coverage-data-pvc",
+			},
+		},
+	})
+
+	k3kSpec.Containers[0].VolumeMounts = append(k3kSpec.Containers[0].VolumeMounts, v1.VolumeMount{
+		Name:      "tmp-covdata",
+		MountPath: "/tmp/covdata",
+	})
+
+	k3kSpec.Containers[0].Env = append(k3kSpec.Containers[0].Env, v1.EnvVar{
+		Name:  "GOCOVERDIR",
+		Value: "/tmp/covdata",
+	})
+
+	k3kDeployment.Spec.Template.Spec = k3kSpec
+
+	_, err = clientset.AppsV1().Deployments(k3kNamespace).Update(ctx, k3kDeployment, metav1.UpdateOptions{})
 	Expect(err).To(Not(HaveOccurred()))
 
 	Eventually(func() bool {
 		GinkgoWriter.Println("Checking K3k deployment status")
 
-		dep, err := clientset.AppsV1().Deployments(k3kNamespace).Get(ctx, k3kName, metav1.GetOptions{})
+		dep, err := clientset.AppsV1().Deployments(k3kNamespace).Get(ctx, k3kDeployment.Name, metav1.GetOptions{})
 		Expect(err).To(Not(HaveOccurred()))
 
 		// 1. Check if the controller has observed the latest generation
@@ -301,7 +301,7 @@ func patchPVC(ctx context.Context, clientset *kubernetes.Clientset) {
 
 		// 3. Check if all updated replicas are available
 		if dep.Status.AvailableReplicas < dep.Status.UpdatedReplicas {
-			GinkgoWriter.Printf("K3k deployment availabl replicas: %d, updated replicas: %d\n", dep.Status.AvailableReplicas, dep.Status.UpdatedReplicas)
+			GinkgoWriter.Printf("K3k deployment available replicas: %d, updated replicas: %d\n", dep.Status.AvailableReplicas, dep.Status.UpdatedReplicas)
 			return false
 		}
 
@@ -340,53 +340,108 @@ var _ = AfterSuite(func() {
 // dumpK3kCoverageData will kill the K3k controller container to force it to dump the coverage data.
 // It will then download the files with kubectl cp into the specified folder. If the folder doesn't exists it will be created.
 func dumpK3kCoverageData(ctx context.Context, folder string) {
-	GinkgoWriter.Println("Restarting k3k controller...")
+	By("Restarting k3k controller")
 
-	var podList corev1.PodList
+	var podList v1.PodList
 
 	err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: k3kNamespace})
 	Expect(err).To(Not(HaveOccurred()))
 
 	k3kPod := podList.Items[0]
+	k3kContainerName := k3kPod.Spec.Containers[0].Name
 
-	cmd := exec.Command("kubectl", "exec", "-n", k3kNamespace, k3kPod.Name, "-c", "k3k", "--", "kill", "1")
+	By("Restarting k3k controller " + k3kPod.Name + "/" + k3kContainerName)
+
+	cmd := exec.Command("kubectl", "exec", "-n", k3kNamespace, k3kPod.Name, "-c", k3kContainerName, "--", "/bin/sh", "-c", "kill 1")
 	output, err := cmd.CombinedOutput()
 	Expect(err).NotTo(HaveOccurred(), string(output))
 
-	Eventually(func() corev1.PodPhase {
-		var pod corev1.Pod
+	By("Waiting to be ready again")
 
+	Eventually(func(g Gomega) {
 		key := types.NamespacedName{
 			Namespace: k3kNamespace,
 			Name:      k3kPod.Name,
 		}
-		err = k8sClient.Get(ctx, key, &pod)
-		Expect(err).To(Not(HaveOccurred()))
 
-		GinkgoWriter.Printf("K3k controller status: %s\n", pod.Status.Phase)
+		var controllerPod v1.Pod
 
-		return pod.Status.Phase
+		err = k8sClient.Get(ctx, key, &controllerPod)
+		g.Expect(err).To(Not(HaveOccurred()))
+
+		_, cond := pod.GetPodCondition(&controllerPod.Status, v1.PodReady)
+		g.Expect(cond).NotTo(BeNil())
+		g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
 	}).
 		MustPassRepeatedly(5).
 		WithPolling(time.Second * 2).
+		WithTimeout(time.Minute * 2).
+		Should(Succeed())
+
+	By("Controller is ready again, dumping coverage data")
+
+	GinkgoWriter.Printf("Downloading covdata from k3k controller %s/%s to %s\n", k3kNamespace, k3kPod.Name, folder)
+
+	tarPod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "tar",
+			Namespace: k3kNamespace,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{{
+				Name:    "tar",
+				Image:   "busybox",
+				Command: []string{"/bin/sh", "-c", "sleep 3600"},
+				VolumeMounts: []v1.VolumeMount{{
+					Name:      "tmp-covdata",
+					MountPath: "/tmp/covdata",
+				}},
+			}},
+			Volumes: []v1.Volume{{
+				Name: "tmp-covdata",
+				VolumeSource: v1.VolumeSource{
+					PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+						ClaimName: "coverage-data-pvc",
+					},
+				},
+			}},
+		},
+	}
+
+	_, err = k8s.CoreV1().Pods(k3kNamespace).Create(ctx, tarPod, metav1.CreateOptions{})
+	Expect(err).To(Not(HaveOccurred()))
+
+	By("Waiting for tar pod to be ready")
+
+	Eventually(func(g Gomega) {
+		err = k8sClient.Get(ctx, client.ObjectKeyFromObject(tarPod), tarPod)
+		g.Expect(err).To(Not(HaveOccurred()))
+
+		_, cond := pod.GetPodCondition(&tarPod.Status, v1.PodReady)
+		g.Expect(cond).NotTo(BeNil())
+		g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+	}).
+		WithPolling(time.Second).
 		WithTimeout(time.Minute).
-		Should(Equal(corev1.PodRunning))
+		Should(Succeed())
 
-	GinkgoWriter.Printf("Downloading covdata from k3k controller to %s\n", folder)
+	By("Copying covdata from tar pod")
 
-	cmd = exec.Command("kubectl", "cp", fmt.Sprintf("%s/%s:/tmp/covdata", k3kNamespace, k3kPod.Name), folder)
+	cmd = exec.Command("kubectl", "cp", fmt.Sprintf("%s/%s:/tmp/covdata", k3kNamespace, tarPod.Name), folder)
 	output, err = cmd.CombinedOutput()
 	Expect(err).NotTo(HaveOccurred(), string(output))
+
+	Expect(k8sClient.Delete(ctx, tarPod)).To(Succeed())
 }
 
 func getK3kLogs(ctx context.Context) io.ReadCloser {
-	var podList corev1.PodList
+	var podList v1.PodList
 
 	err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: k3kNamespace})
 	Expect(err).To(Not(HaveOccurred()))
 
 	k3kPod := podList.Items[0]
-	req := k8s.CoreV1().Pods(k3kPod.Namespace).GetLogs(k3kPod.Name, &corev1.PodLogOptions{Previous: true})
+	req := k8s.CoreV1().Pods(k3kPod.Namespace).GetLogs(k3kPod.Name, &v1.PodLogOptions{Previous: true})
 	podLogs, err := req.Stream(ctx)
 	Expect(err).To(Not(HaveOccurred()))
 
@@ -427,13 +482,13 @@ func podExec(ctx context.Context, clientset *kubernetes.Clientset, config *rest.
 		SubResource("exec")
 	scheme := runtime.NewScheme()
 
-	if err := corev1.AddToScheme(scheme); err != nil {
+	if err := v1.AddToScheme(scheme); err != nil {
 		return nil, fmt.Errorf("error adding to scheme: %v", err)
 	}
 
 	parameterCodec := runtime.NewParameterCodec(scheme)
 
-	req.VersionedParams(&corev1.PodExecOptions{
+	req.VersionedParams(&v1.PodExecOptions{
 		Command: command,
 		Stdin:   stdin != nil,
 		Stdout:  stdout != nil,
@@ -461,8 +516,8 @@ func podExec(ctx context.Context, clientset *kubernetes.Clientset, config *rest.
 	return stderr.Bytes(), nil
 }
 
-func caCertSecret(name, namespace string, crt, key []byte) *corev1.Secret {
-	return &corev1.Secret{
+func caCertSecret(name, namespace string, crt, key []byte) *v1.Secret {
+	return &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: namespace,