fix for default priorityClasses (#182)

* Add pvc syncing support

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fixes

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

---------

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

.drone.yml

@@ -1,162 +0,0 @@
----
-kind: pipeline
-name: amd64
-
-platform:
-  os: linux
-  arch: amd64
-
-steps:
-- name: build
-  image: rancher/dapper:v0.6.0
-  environment:
-    CROSS: 'true'
-    GITHUB_TOKEN:
-      from_secret: github_token
-  commands:
-  - dapper ci
-  - echo "${DRONE_TAG}-amd64" | sed -e 's/+/-/g' >.tags
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-  when:
-    ref:
-      exclude:
-        - refs/tags/chart-*
-
-- name: package-chart
-  image: rancher/dapper:v0.6.0
-  environment:
-    GITHUB_TOKEN:
-      from_secret: github_token
-  commands:
-  - dapper package-chart
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-  when:
-    ref:
-    - refs/tags/chart-*
-    instance:
-    - drone-publish.rancher.io
-    event:
-    - tag
-
-- name: release-chart
-  image: plugins/github-release
-  settings:
-    api_key:
-      from_secret: github_token
-    checksum:
-    - sha256
-    checksum_file: CHECKSUMsum.txt
-    checksum_flatten: true
-    files:
-    - "deploy/*"
-  when:
-    instance:
-    - drone-publish.rancher.io
-    ref:
-      include:
-      - refs/tags/chart-*  
-    event:
-    - tag
-
-- name: index-chart
-  image: rancher/dapper:v0.6.0
-  environment:
-    GITHUB_TOKEN:
-      from_secret: github_token
-  commands:
-  - dapper index-chart
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-  when:
-    ref:
-    - refs/tags/chart-*
-    instance:
-    - drone-publish.rancher.io
-    event:
-    - tag
-
-- name: github_binary_release
-  image: plugins/github-release
-  settings:
-    api_key:
-      from_secret: github_token
-    prerelease: true
-    checksum:
-    - sha256
-    checksum_file: CHECKSUMsum-amd64.txt
-    checksum_flatten: true
-    files:
-    - "bin/*"
-  when:
-    instance:
-    - drone-publish.rancher.io
-    ref:
-      include:
-      - refs/head/master
-      - refs/tags/*
-      exclude:
-      - refs/tags/chart-*  
-    event:
-    - tag
-
-- name: docker-publish
-  image: plugins/docker
-  settings:
-    dockerfile: package/Dockerfile
-    password:
-      from_secret: docker_password
-    repo: "rancher/k3k"
-    username:
-      from_secret: docker_username
-  when:
-    instance:
-    - drone-publish.rancher.io
-    ref:
-      include:
-      - refs/head/master
-      - refs/tags/*
-      exclude:
-      - refs/tags/chart-*  
-    event:
-    - tag
-
-volumes:
-- name: docker
-  host:
-    path: /var/run/docker.sock
----
-kind: pipeline
-type: docker
-name: manifest
-
-platform:
-  os: linux
-  arch: amd64
-
-steps:
-  - name: push-runtime-manifest
-    image: plugins/manifest
-    settings:
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-      spec: manifest-runtime.tmpl
-    when:
-      event:
-        - tag
-      instance:
-        - drone-publish.rancher.io
-      ref:
-        include:
-        - refs/head/master
-        - refs/tags/*
-        exclude:
-        - refs/tags/chart-*  
-depends_on:
-  - amd64

.github/workflows/build.yml

@@ -0,0 +1,34 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+    contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Run GoReleaser
+      uses: goreleaser/goreleaser-action@v6
+      with:
+        distribution: goreleaser
+        version: v2
+        args: --clean --snapshot
+      env:
+        REPO: ${{ github.repository }}
+        REGISTRY:
+        

.github/workflows/chart.yml

@@ -0,0 +1,31 @@
+on:
+  push:
+    tags:
+    - "chart-*"
+
+env:
+    GH_TOKEN: ${{ github.token }}
+
+name: Chart
+permissions:
+    contents: write
+    id-token: write
+jobs:
+  chart-release:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Package Chart
+      run: |
+        make package-chart;
+
+    - name: Release Chart
+      run: |
+        gh release upload ${{ github.ref_name }} deploy/*
+      
+    - name: Index Chart
+      run: |
+        make index-chart
+        

.github/workflows/release-delete.yml

@@ -0,0 +1,61 @@
+name: Release - Delete Draft
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        type: string
+        description: The tag of the release
+
+permissions:
+    contents: write
+    packages: write
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+jobs:
+  release-delete:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check tag
+      if: inputs.tag == ''
+      run: echo "::error::Missing tag from input" && exit 1
+    
+    - name: Checkout code
+      uses: actions/checkout@v4
+    
+    - name: Check if release is draft
+      run: |
+        CURRENT_TAG=${{ inputs.tag }}
+        isDraft=$(gh release view ${CURRENT_TAG} --json isDraft --jq ".isDraft")
+        if [ "$isDraft" = true ]; then
+          echo "Release ${CURRENT_TAG} is draft"
+        else
+          echo "::error::Cannot delete non-draft release" && exit 1
+        fi
+
+    - name: Delete packages from Github Container Registry
+      run: |
+        CURRENT_TAG=${{ inputs.tag }}
+        echo "Deleting packages with tag ${CURRENT_TAG}"
+
+        JQ_QUERY=".[] | select(.metadata.container.tags[] == \"${CURRENT_TAG}\")"
+
+        for package in k3k k3k-kubelet
+        do
+          echo "Deleting ${package} image"
+          PACKAGE_TO_DELETE=$(gh api /user/packages/container/${package}/versions --jq "${JQ_QUERY}")
+          echo $PACKAGE_TO_DELETE | jq
+
+          PACKAGE_ID=$(echo $PACKAGE_TO_DELETE | jq .id)
+          echo "Deleting ${PACKAGE_ID}"
+          gh api --method DELETE /user/packages/container/${package}/versions/${PACKAGE_ID}  
+        done
+
+    - name: Delete Github release
+      run: |
+        CURRENT_TAG=${{ inputs.tag }}
+        echo "Deleting release ${CURRENT_TAG}"
+        gh release delete ${CURRENT_TAG}

.github/workflows/release.yml

@@ -0,0 +1,87 @@
+name: Release
+
+on:
+  push:
+    tags:
+    - "v*"
+  workflow_dispatch:
+    inputs:
+      commit:
+        type: string
+        description: Checkout a specific commit
+
+permissions:
+    contents: write
+    packages: write
+    id-token: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+    
+    - name: Checkout code at the specific commit
+      if: inputs.commit != ''
+      run: git checkout ${{ inputs.commit }}
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: "Read secrets"
+      uses: rancher-eio/read-vault-secrets@main
+      if: github.repository_owner == 'rancher'
+      with:
+        secrets: |
+          secret/data/github/repo/${{ github.repository }}/dockerhub/${{ github.repository_owner }}/credentials username | DOCKER_USERNAME ;
+          secret/data/github/repo/${{ github.repository }}/dockerhub/${{ github.repository_owner }}/credentials password | DOCKER_PASSWORD ;
+
+    # Manually dispatched workflows (or forks) will use ghcr.io
+    - name: Setup ghcr.io
+      if: github.event_name == 'workflow_dispatch' || github.repository_owner != 'rancher'
+      run: |
+        echo "REGISTRY=ghcr.io"                     >> $GITHUB_ENV
+        echo "DOCKER_USERNAME=${{ github.actor }}"  >> $GITHUB_ENV
+        echo "DOCKER_PASSWORD=${{ github.token }}"  >> $GITHUB_ENV
+
+    - name: Login to container registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ env.DOCKER_USERNAME }}
+        password: ${{ env.DOCKER_PASSWORD }}
+    
+    # If the tag does not exists the workflow was manually triggered.
+    # That means we are creating temporary nightly builds, with a "fake" local tag
+    - name: Check release tag
+      id: release-tag
+      run: |
+        CURRENT_TAG=$(git describe --tag --always)
+
+        if git show-ref --tags ${CURRENT_TAG} --quiet; then
+          echo "tag ${CURRENT_TAG} already exists";
+        else
+          echo "tag ${CURRENT_TAG} does not exist"
+          git tag ${CURRENT_TAG}
+        fi
+
+        echo "CURRENT_TAG=${CURRENT_TAG}" >> "$GITHUB_OUTPUT"
+
+    - name: Run GoReleaser
+      uses: goreleaser/goreleaser-action@v6
+      with:
+        distribution: goreleaser
+        version: v2
+        args: --clean
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+        GORELEASER_CURRENT_TAG: ${{ steps.release-tag.outputs.CURRENT_TAG }}
+        REGISTRY: ${{ env.REGISTRY }}
+        REPO: ${{ github.repository }}

.github/workflows/test.yaml

@@ -0,0 +1,41 @@
+name: Tests
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+    contents: read
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+    
+    - name: Check go modules
+      run: |
+        go mod tidy
+        test -z "$(git status --porcelain)"
+
+    - name: Install tools
+      run: |
+        go install github.com/onsi/ginkgo/v2/ginkgo
+
+        # With Golang 1.22 we need to use the release-0.18 branch
+        go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.18
+
+        ENVTEST_BIN=$(setup-envtest use -p path)
+        sudo mkdir -p /usr/local/kubebuilder/bin
+        sudo cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin
+
+    - name: Run tests
+      run: |
+        ginkgo run ./...

.gitignore

@@ -4,4 +4,6 @@
 /dist
 *.swp
 .idea
-
+.vscode/
+__debug*
+*-kubeconfig.yaml

.goreleaser.yaml

@@ -0,0 +1,90 @@
+version: 2
+
+release:
+  draft: true
+  replace_existing_draft: true
+  prerelease: auto
+
+before:
+  hooks:
+    - go mod tidy
+    - go generate ./...
+
+builds:
+  - id: k3k
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - "amd64"
+      - "arm64"
+      - "s390x"
+
+  - id: k3k-kubelet
+    main: ./k3k-kubelet
+    binary: k3k-kubelet
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - "amd64"
+      - "arm64"
+      - "s390x"
+  
+  - id: k3kcli
+    main: ./cli
+    binary: k3kcli
+    env:
+      - CGO_ENABLED=0
+    goarch:
+      - "amd64"
+      - "arm64"
+
+archives:
+  - format: binary
+    name_template: >-
+      {{ .Binary }}-{{- .Os }}-{{ .Arch }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    format_overrides:
+      - goos: windows
+        format: zip
+
+# For the image_templates we are using the following expression to build images for the correct registry
+#    {{- if .Env.REGISTRY }}{{ .Env.REGISTRY }}/{{ end }}
+#
+# REGISTRY=        -> rancher/k3k:vX.Y.Z
+# REGISTRY=ghcr.io -> ghcr.io/rancher/k3k:latest:vX.Y.Z
+#
+dockers:
+  - id: k3k
+    use: docker
+    ids:
+      - k3k
+      - k3kcli
+    dockerfile: "package/Dockerfile"
+    skip_push: false
+    image_templates:
+      - "{{- if .Env.REGISTRY }}{{ .Env.REGISTRY }}/{{ end }}{{ .Env.REPO }}:{{ .Tag }}"
+    build_flag_templates:
+      - "--build-arg=BIN_K3K=k3k"
+      - "--build-arg=BIN_K3KCLI=k3kcli"
+
+  - id: k3k-kubelet
+    use: docker
+    ids:
+      - k3k-kubelet
+    dockerfile: "package/Dockerfile.kubelet"
+    skip_push: false
+    image_templates:
+      - "{{- if .Env.REGISTRY }}{{ .Env.REGISTRY }}/{{ end }}{{ .Env.REPO }}-kubelet:{{ .Tag }}"
+    build_flag_templates:
+      - "--build-arg=BIN_K3K_KUBELET=k3k-kubelet"
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - "^docs:"
+      - "^test:"

Dockerfile.dapper

@@ -1,4 +1,4 @@
-ARG GOLANG=rancher/hardened-build-base:v1.20.7b2
+ARG GOLANG=rancher/hardened-build-base:v1.22.2b1
 FROM ${GOLANG}
 
 ARG DAPPER_HOST_ARCH
@@ -6,16 +6,28 @@ ENV ARCH $DAPPER_HOST_ARCH
 
 RUN apk -U add \bash git gcc musl-dev docker vim less file curl wget ca-certificates
 RUN if [ "${ARCH}" == "amd64" ]; then \
-        curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.15.0; \
+        curl -sL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.59.0; \
     fi
 
 RUN curl -sL https://github.com/helm/chart-releaser/releases/download/v1.5.0/chart-releaser_1.5.0_linux_${ARCH}.tar.gz | tar -xz cr \
     && mv cr /bin/
 
+# Tool for CRD generation.
+ENV CONTROLLER_GEN_VERSION v0.14.0
+RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_GEN_VERSION}
+
+# Tool to setup the envtest framework to run the controllers integration tests
+# Note: With Golang 1.22 we need to use the release-0.18 branch
+ENV SETUP_ENVTEST_VERSION release-0.18
+RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@${SETUP_ENVTEST_VERSION} && \
+    ENVTEST_BIN=$(setup-envtest use -p path) && \
+    mkdir -p /usr/local/kubebuilder/bin && \
+    cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin
+
 ENV GO111MODULE on
 ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS GITHUB_TOKEN
 ENV DAPPER_SOURCE /go/src/github.com/rancher/k3k/
-ENV DAPPER_OUTPUT ./bin ./dist ./deploy
+ENV DAPPER_OUTPUT ./bin ./dist ./deploy ./charts
 ENV DAPPER_DOCKER_SOCKET true
 ENV HOME ${DAPPER_SOURCE}
 WORKDIR ${DAPPER_SOURCE}

Makefile

@@ -1,5 +1,4 @@
 TARGETS := $(shell ls ops)
-
 .dapper:
 	@echo Downloading dapper
 	@curl -sL https://releases.rancher.com/dapper/latest/dapper-$$(uname -s)-$$(uname -m) > .dapper.tmp
@@ -12,4 +11,4 @@ $(TARGETS): .dapper
 
 .DEFAULT_GOAL := default
 
-.PHONY: $(TARGETS)
+.PHONY: $(TARGETS)

README.md

@@ -1,7 +1,13 @@
 # K3K
 
+[![Experimental](https://img.shields.io/badge/status-experimental-orange.svg)](https://shields.io/)
+
 A Kubernetes in Kubernetes tool, k3k provides a way to run multiple embedded isolated k3s clusters on your kubernetes cluster.
 
+**Experimental Tool**
+
+This project is still under development and is considered experimental. It may have limitations, bugs, or changes. Please use with caution and report any issues you encounter. We appreciate your feedback as we continue to refine and improve this tool.
+
 ## Example
 
 An example on creating a k3k cluster on an RKE2 host using k3kcli
@@ -120,3 +126,18 @@ To create a new cluster you can use:
 ```sh
 k3k cluster create --name example-cluster --token test
 ```
+
+
+## Tests
+
+To run the tests we use [Ginkgo](https://onsi.github.io/ginkgo/), and [`envtest`](https://book.kubebuilder.io/reference/envtest) for testing the controllers.
+
+Install the required binaries from `envtest` with [`setup-envtest`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/tools/setup-envtest), and then put them in the default path `/usr/local/kubebuilder/bin`:
+
+```
+ENVTEST_BIN=$(setup-envtest use -p path)
+sudo mkdir -p /usr/local/kubebuilder/bin
+sudo cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin
+```
+
+then run `ginkgo run ./...`.

charts/k3k/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: k3k
 description: A Helm chart for K3K
 type: application
-version: 0.1.1-r1
-appVersion: 0.1.0
+version: 0.1.5-r5
+appVersion: 0.2.0

charts/k3k/crds/cluster.yaml

@@ -1,116 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: clusters.k3k.io
-spec:
-  group: k3k.io
-  versions:
-    - name: v1alpha1
-      served: true
-      storage: true
-      schema:
-        openAPIV3Schema:
-          type: object
-          properties:
-            spec:
-              type: object
-              properties:
-                name:
-                  type: string
-                version:
-                  type: string
-                servers:
-                  type: integer
-                agents:
-                  type: integer
-                token:
-                  type: string
-                clusterCIDR:
-                  type: string
-                serviceCIDR:
-                  type: string
-                clusterDNS:
-                  type: string
-                serverArgs:
-                  type: array
-                  items:
-                    type: string
-                agentArgs:
-                  type: array
-                  items:
-                    type: string
-                tlsSANs:
-                  type: array
-                  items:
-                    type: string
-                persistence:
-                  type: object
-                  properties:
-                    type:
-                      type: string
-                      default: "ephermal"
-                    storageClassName:
-                      type: string
-                    storageRequestSize:
-                      type: string
-                addons:
-                  type: array
-                  items:
-                    type: object
-                    properties:
-                      secretNamespace:
-                        type: string
-                      secretRef:
-                        type: string
-                expose:
-                  type: object
-                  properties:
-                    ingress:
-                      type: object
-                      properties:
-                        enabled: 
-                          type: boolean
-                        ingressClassName:
-                          type: string
-                    loadbalancer:
-                      type: object
-                      properties:
-                        enabled: 
-                          type: boolean
-                    nodePort:
-                      type: object
-                      properties:
-                        enabled: 
-                          type: boolean
-            status:
-              type: object
-              properties:
-                overrideClusterCIDR:
-                  type: boolean
-                clusterCIDR:
-                  type: string
-                overrideServiceCIDR:
-                  type: boolean
-                serviceCIDR:
-                  type: string
-                clusterDNS:
-                  type: string
-                tlsSANs:
-                  type: array
-                  items:
-                    type: string
-                persistence:
-                  type: object
-                  properties:
-                    type:
-                      type: string
-                      default: "ephermal"
-                    storageClassName:
-                      type: string
-                    storageRequestSize:
-                      type: string
-  scope: Cluster
-  names:
-    plural: clusters
-    singular: cluster
-    kind: Cluster

charts/k3k/crds/k3k.io_clusters.yaml

@@ -0,0 +1,259 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  name: clusters.k3k.io
+spec:
+  group: k3k.io
+  names:
+    kind: Cluster
+    listKind: ClusterList
+    plural: clusters
+    singular: cluster
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              addons:
+                description: Addons is a list of secrets containing raw YAML which
+                  will be deployed in the virtual K3k cluster on startup.
+                items:
+                  properties:
+                    secretNamespace:
+                      type: string
+                    secretRef:
+                      type: string
+                  type: object
+                type: array
+              agentArgs:
+                description: AgentArgs are the ordered key value pairs (e.x. "testArg",
+                  "testValue") for the K3s pods running in agent mode.
+                items:
+                  type: string
+                type: array
+              agents:
+                description: Agents is the number of K3s pods to run in agent (worker)
+                  mode.
+                format: int32
+                type: integer
+                x-kubernetes-validations:
+                - message: invalid value for agents
+                  rule: self >= 0
+              clusterCIDR:
+                description: ClusterCIDR is the CIDR range for the pods of the cluster.
+                  Defaults to 10.42.0.0/16.
+                type: string
+                x-kubernetes-validations:
+                - message: clusterCIDR is immutable
+                  rule: self == oldSelf
+              clusterDNS:
+                description: |-
+                  ClusterDNS is the IP address for the coredns service. Needs to be in the range provided by ServiceCIDR or CoreDNS may not deploy.
+                  Defaults to 10.43.0.10.
+                type: string
+                x-kubernetes-validations:
+                - message: clusterDNS is immutable
+                  rule: self == oldSelf
+              clusterLimit:
+                description: Limit is the limits that apply for the server/worker
+                  nodes.
+                properties:
+                  serverLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: ServerLimit is the limits (cpu/mem) that apply to
+                      the server nodes
+                    type: object
+                  workerLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: WorkerLimit is the limits (cpu/mem) that apply to
+                      the agent nodes
+                    type: object
+                type: object
+              expose:
+                description: |-
+                  Expose contains options for exposing the apiserver inside/outside of the cluster. By default, this is only exposed as a
+                  clusterIP which is relatively secure, but difficult to access outside of the cluster.
+                properties:
+                  ingress:
+                    properties:
+                      enabled:
+                        type: boolean
+                      ingressClassName:
+                        type: string
+                    type: object
+                  loadbalancer:
+                    properties:
+                      enabled:
+                        type: boolean
+                    required:
+                    - enabled
+                    type: object
+                  nodePort:
+                    properties:
+                      enabled:
+                        type: boolean
+                    required:
+                    - enabled
+                    type: object
+                type: object
+              mode:
+                allOf:
+                - enum:
+                  - shared
+                  - virtual
+                - enum:
+                  - shared
+                  - virtual
+                default: shared
+                description: Mode is the cluster provisioning mode which can be either
+                  "shared" or "virtual". Defaults to "shared"
+                type: string
+                x-kubernetes-validations:
+                - message: mode is immutable
+                  rule: self == oldSelf
+              nodeSelector:
+                additionalProperties:
+                  type: string
+                description: |-
+                  NodeSelector is the node selector that will be applied to all server/agent pods.
+                  In "shared" mode the node selector will be applied also to the workloads.
+                type: object
+              persistence:
+                description: |-
+                  Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data
+                  persistence is guaranteed, so restart of a virtual cluster pod may result in data loss without this field.
+                properties:
+                  storageClassName:
+                    type: string
+                  storageRequestSize:
+                    type: string
+                  type:
+                    default: ephemeral
+                    description: Type can be ephermal, static, dynamic
+                    type: string
+                required:
+                - type
+                type: object
+              priorityClass:
+                description: |-
+                  PriorityClass is the priorityClassName that will be applied to all server/agent pods.
+                  In "shared" mode the priorityClassName will be applied also to the workloads.
+                type: string
+              serverArgs:
+                description: ServerArgs are the ordered key value pairs (e.x. "testArg",
+                  "testValue") for the K3s pods running in server mode.
+                items:
+                  type: string
+                type: array
+              servers:
+                description: Servers is the number of K3s pods to run in server (controlplane)
+                  mode.
+                format: int32
+                type: integer
+                x-kubernetes-validations:
+                - message: cluster must have at least one server
+                  rule: self >= 1
+              serviceCIDR:
+                description: ServiceCIDR is the CIDR range for the services in the
+                  cluster. Defaults to 10.43.0.0/16.
+                type: string
+                x-kubernetes-validations:
+                - message: serviceCIDR is immutable
+                  rule: self == oldSelf
+              tlsSANs:
+                description: TLSSANs are the subjectAlternativeNames for the certificate
+                  the K3s server will use.
+                items:
+                  type: string
+                type: array
+              tokenSecretRef:
+                description: |-
+                  TokenSecretRef is Secret reference used as a token join server and worker nodes to the cluster. The controller
+                  assumes that the secret has a field "token" in its data, any other fields in the secret will be ignored.
+                properties:
+                  name:
+                    description: name is unique within a namespace to reference a
+                      secret resource.
+                    type: string
+                  namespace:
+                    description: namespace defines the space within which the secret
+                      name must be unique.
+                    type: string
+                type: object
+                x-kubernetes-map-type: atomic
+              version:
+                description: Version is a string representing the Kubernetes version
+                  to be used by the virtual nodes.
+                type: string
+            required:
+            - agents
+            - mode
+            - servers
+            - version
+            type: object
+          status:
+            properties:
+              clusterCIDR:
+                type: string
+              clusterDNS:
+                type: string
+              persistence:
+                properties:
+                  storageClassName:
+                    type: string
+                  storageRequestSize:
+                    type: string
+                  type:
+                    default: ephemeral
+                    description: Type can be ephermal, static, dynamic
+                    type: string
+                required:
+                - type
+                type: object
+              serviceCIDR:
+                type: string
+              tlsSANs:
+                items:
+                  type: string
+                type: array
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

charts/k3k/crds/k3k.io_clustersets.yaml

@@ -0,0 +1,210 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  name: clustersets.k3k.io
+spec:
+  group: k3k.io
+  names:
+    kind: ClusterSet
+    listKind: ClusterSetList
+    plural: clustersets
+    singular: clusterset
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            default: {}
+            description: Spec is the spec of the ClusterSet
+            properties:
+              allowedNodeTypes:
+                default:
+                - shared
+                description: AllowedNodeTypes are the allowed cluster provisioning
+                  modes. Defaults to [shared].
+                items:
+                  description: ClusterMode is the possible provisioning mode of a
+                    Cluster.
+                  enum:
+                  - shared
+                  - virtual
+                  type: string
+                minItems: 1
+                type: array
+                x-kubernetes-validations:
+                - message: mode is immutable
+                  rule: self == oldSelf
+              defaultLimits:
+                description: DefaultLimits are the limits used for servers/agents
+                  when a cluster in the set doesn't provide any
+                properties:
+                  serverLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: ServerLimit is the limits (cpu/mem) that apply to
+                      the server nodes
+                    type: object
+                  workerLimit:
+                    additionalProperties:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    description: WorkerLimit is the limits (cpu/mem) that apply to
+                      the agent nodes
+                    type: object
+                type: object
+              defaultNodeSelector:
+                additionalProperties:
+                  type: string
+                description: DefaultNodeSelector is the node selector that applies
+                  to all clusters (server + agent) in the set
+                type: object
+              defaultPriorityClass:
+                description: DefaultPriorityClass is the priorityClassName applied
+                  to all pods of all clusters in the set
+                type: string
+              disableNetworkPolicy:
+                description: DisableNetworkPolicy is an option that will disable the
+                  creation of a default networkpolicy for cluster isolation
+                type: boolean
+              maxLimits:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: MaxLimits are the limits that apply to all clusters (server
+                  + agent) in the set
+                type: object
+              podSecurityAdmissionLevel:
+                description: PodSecurityAdmissionLevel is the policy level applied
+                  to the pods in the namespace.
+                enum:
+                - privileged
+                - baseline
+                - restricted
+                type: string
+            type: object
+          status:
+            description: Status is the status of the ClusterSet
+            properties:
+              conditions:
+                description: Conditions are the invidual conditions for the cluster
+                  set
+                items:
+                  description: "Condition contains details for one aspect of the current
+                    state of this API Resource.\n---\nThis struct is intended for
+                    direct use as an array at the field path .status.conditions.  For
+                    example,\n\n\n\ttype FooStatus struct{\n\t    // Represents the
+                    observations of a foo's current state.\n\t    // Known .status.conditions.type
+                    are: \"Available\", \"Progressing\", and \"Degraded\"\n\t    //
+                    +patchMergeKey=type\n\t    // +patchStrategy=merge\n\t    // +listType=map\n\t
+                    \   // +listMapKey=type\n\t    Conditions []metav1.Condition `json:\"conditions,omitempty\"
+                    patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t
+                    \   // other fields\n\t}"
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: |-
+                        type of condition in CamelCase or in foo.example.com/CamelCase.
+                        ---
+                        Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be
+                        useful (see .node.status.conditions), the ability to deconflict is important.
+                        The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+              lastUpdateTime:
+                description: LastUpdate is the timestamp when the status was last
+                  updated
+                type: string
+              observedGeneration:
+                description: ObservedGeneration was the generation at the time the
+                  status was updated.
+                format: int64
+                type: integer
+              summary:
+                description: Summary is a summary of the status
+                type: string
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

charts/k3k/templates/deployment.yaml

@@ -19,8 +19,16 @@ spec:
         - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           name: {{ .Chart.Name }}
+          env:
+          - name: CLUSTER_CIDR
+            value: {{ .Values.host.clusterCIDR }}
+          - name: SHARED_AGENT_IMAGE
+            value: "{{ .Values.sharedAgent.image.repository }}:{{ .Values.sharedAgent.image.tag }}"
           ports:
           - containerPort: 8080
             name: https
             protocol: TCP
-      serviceAccountName: {{ include "k3k.serviceAccountName" . }}
+          - containerPort: 9443
+            name: https-webhook
+            protocol: TCP
+      serviceAccountName: {{ include "k3k.serviceAccountName" . }}

charts/k3k/templates/rbac.yaml

@@ -11,4 +11,27 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: {{ include "k3k.serviceAccountName" . }}
-    namespace: {{ .Values.namespace }}
+    namespace: {{ .Values.namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "k3k.fullname" . }}-node-proxy
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - "nodes"
+  - "nodes/proxy"
+  verbs:
+  - "get"
+  - "list"
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "k3k.fullname" . }}-node-proxy
+roleRef:
+  kind: ClusterRole
+  name: {{ include "k3k.fullname" . }}-node-proxy
+  apiGroup: rbac.authorization.k8s.io

charts/k3k/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata: 
+  name: k3k-webhook
+  labels:
+    {{- include "k3k.labels" . | nindent 4 }}
+  namespace: {{ .Values.namespace }}
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      name: https-webhook
+      targetPort: 9443
+  selector:
+    {{- include "k3k.selectorLabels" . | nindent 6 }}

charts/k3k/values.yaml

@@ -2,18 +2,29 @@ replicaCount: 1
 namespace: k3k-system
 
 image:
-  repository: rancher/k3k
+  repository: rancher/k3k 
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "v0.1.0-amd64"
+  tag: "v0.2.1"
 
 imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
 
+host:
+# clusterCIDR specifies the clusterCIDR that will be added to the default networkpolicy for clustersets, if not set
+# the controller will collect the PodCIDRs of all the nodes on the system.
+  clusterCIDR: ""
+
 serviceAccount:
   # Specifies whether a service account should be created
   create: true
   # The name of the service account to use.
   # If not set and create is true, a name is generated using the fullname template
   name: ""
+
+# configuration related to the shared agent mode in k3k
+sharedAgent:
+  image:
+    repository: "rancher/k3k"
+    tag: "k3k-kubelet-dev"

cli/cmds/cluster/cluster.go

@@ -5,21 +5,29 @@ import (
 	"github.com/urfave/cli"
 )
 
-var clusterSubcommands = []cli.Command{
+var subcommands = []cli.Command{
 	{
 		Name:            "create",
 		Usage:           "Create new cluster",
 		SkipFlagParsing: false,
 		SkipArgReorder:  true,
-		Action:          createCluster,
+		Action:          create,
 		Flags:           append(cmds.CommonFlags, clusterCreateFlags...),
 	},
+	{
+		Name:            "delete",
+		Usage:           "Delete an existing cluster",
+		SkipFlagParsing: false,
+		SkipArgReorder:  true,
+		Action:          delete,
+		Flags:           append(cmds.CommonFlags, clusterDeleteFlags...),
+	},
 }
 
-func NewClusterCommand() cli.Command {
+func NewCommand() cli.Command {
 	return cli.Command{
 		Name:        "cluster",
 		Usage:       "cluster command",
-		Subcommands: clusterSubcommands,
+		Subcommands: subcommands,
 	}
 }

cli/cmds/cluster/create.go

@@ -3,7 +3,6 @@ package cluster
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -12,34 +11,25 @@ import (
 
 	"github.com/rancher/k3k/cli/cmds"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	"github.com/rancher/k3k/pkg/controller/cluster"
+	"github.com/rancher/k3k/pkg/controller"
+	k3kcluster "github.com/rancher/k3k/pkg/controller/cluster"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
-	"github.com/rancher/k3k/pkg/controller/util"
+	"github.com/rancher/k3k/pkg/controller/kubeconfig"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authentication/user"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-var (
-	Scheme  = runtime.NewScheme()
-	backoff = wait.Backoff{
-		Steps:    5,
-		Duration: 20 * time.Second,
-		Factor:   2,
-		Jitter:   0.1,
-	}
-)
+var Scheme = runtime.NewScheme()
 
 func init() {
 	_ = clientgoscheme.AddToScheme(Scheme)
@@ -58,6 +48,7 @@ var (
 	persistenceType  string
 	storageClassName string
 	version          string
+	mode             string
 
 	clusterCreateFlags = []cli.Flag{
 		cli.StringFlag{
@@ -118,12 +109,18 @@ var (
 			Destination: &version,
 			Value:       "v1.26.1-k3s1",
 		},
+		cli.StringFlag{
+			Name:        "mode",
+			Usage:       "k3k mode type",
+			Destination: &mode,
+			Value:       "shared",
+		},
 	}
 )
 
-func createCluster(clx *cli.Context) error {
+func create(clx *cli.Context) error {
 	ctx := context.Background()
-	if err := validateCreateFlags(clx); err != nil {
+	if err := validateCreateFlags(); err != nil {
 		return err
 	}
 
@@ -139,9 +136,18 @@ func createCluster(clx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	if token != "" {
+		logrus.Infof("Creating cluster token secret")
+		obj := k3kcluster.TokenSecretObj(token, name, cmds.Namespace())
+		if err := ctrlClient.Create(ctx, &obj); err != nil {
+			return err
+		}
+	}
 	logrus.Infof("Creating a new cluster [%s]", name)
 	cluster := newCluster(
 		name,
+		cmds.Namespace(),
+		mode,
 		token,
 		int32(servers),
 		int32(agents),
@@ -174,14 +180,25 @@ func createCluster(clx *cli.Context) error {
 	}
 
 	logrus.Infof("Extracting Kubeconfig for [%s] cluster", name)
+	cfg := &kubeconfig.KubeConfig{
+		CN:         controller.AdminCommonName,
+		ORG:        []string{user.SystemPrivilegedGroup},
+		ExpiryDate: 0,
+	}
+
+	logrus.Infof("waiting for cluster to be available..")
+
+	// retry every 5s for at most 2m, or 25 times
+	availableBackoff := wait.Backoff{
+		Duration: 5 * time.Second,
+		Cap:      2 * time.Minute,
+		Steps:    25,
+	}
+
 	var kubeconfig []byte
-	if err := retry.OnError(backoff, apierrors.IsNotFound, func() error {
-		kubeconfig, err = extractKubeconfig(ctx, ctrlClient, cluster, host[0])
-		if err != nil {
-			logrus.Infof("waiting for cluster to be available: %v", err)
-			return err
-		}
-		return nil
+	if err := retry.OnError(availableBackoff, apierrors.IsNotFound, func() error {
+		kubeconfig, err = cfg.Extract(ctx, ctrlClient, cluster, host[0])
+		return err
 	}); err != nil {
 		return err
 	}
@@ -200,18 +217,15 @@ func createCluster(clx *cli.Context) error {
 	return os.WriteFile(cluster.Name+"-kubeconfig.yaml", kubeconfig, 0644)
 }
 
-func validateCreateFlags(clx *cli.Context) error {
+func validateCreateFlags() error {
 	if persistenceType != server.EphermalNodesType &&
 		persistenceType != server.DynamicNodesType {
 		return errors.New("invalid persistence type")
 	}
-	if token == "" {
-		return errors.New("empty cluster token")
-	}
 	if name == "" {
 		return errors.New("empty cluster name")
 	}
-	if name == cluster.ClusterInvalidName {
+	if name == k3kcluster.ClusterInvalidName {
 		return errors.New("invalid cluster name")
 	}
 	if servers <= 0 {
@@ -220,22 +234,24 @@ func validateCreateFlags(clx *cli.Context) error {
 	if cmds.Kubeconfig == "" && os.Getenv("KUBECONFIG") == "" {
 		return errors.New("empty kubeconfig")
 	}
+	if mode != "shared" && mode != "virtual" {
+		return errors.New(`mode should be one of "shared" or "virtual"`)
+	}
 
 	return nil
 }
 
-func newCluster(name, token string, servers, agents int32, clusterCIDR, serviceCIDR string, serverArgs, agentArgs []string) *v1alpha1.Cluster {
-	return &v1alpha1.Cluster{
+func newCluster(name, namespace, mode, token string, servers, agents int32, clusterCIDR, serviceCIDR string, serverArgs, agentArgs []string) *v1alpha1.Cluster {
+	cluster := &v1alpha1.Cluster{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: name,
+			Name:      name,
+			Namespace: namespace,
 		},
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Cluster",
 			APIVersion: "k3k.io/v1alpha1",
 		},
 		Spec: v1alpha1.ClusterSpec{
-			Name:        name,
-			Token:       token,
 			Servers:     &servers,
 			Agents:      &agents,
 			ClusterCIDR: clusterCIDR,
@@ -243,90 +259,18 @@ func newCluster(name, token string, servers, agents int32, clusterCIDR, serviceC
 			ServerArgs:  serverArgs,
 			AgentArgs:   agentArgs,
 			Version:     version,
+			Mode:        v1alpha1.ClusterMode(mode),
 			Persistence: &v1alpha1.PersistenceConfig{
 				Type:             persistenceType,
 				StorageClassName: storageClassName,
 			},
 		},
 	}
-}
-
-func extractKubeconfig(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, serverIP string) ([]byte, error) {
-	nn := types.NamespacedName{
-		Name:      cluster.Name + "-kubeconfig",
-		Namespace: util.ClusterNamespace(cluster),
-	}
-
-	var kubeSecret v1.Secret
-	if err := client.Get(ctx, nn, &kubeSecret); err != nil {
-		return nil, err
-	}
-
-	kubeconfig := kubeSecret.Data["kubeconfig.yaml"]
-	if kubeconfig == nil {
-		return nil, errors.New("empty kubeconfig")
-	}
-
-	nn = types.NamespacedName{
-		Name:      "k3k-server-service",
-		Namespace: util.ClusterNamespace(cluster),
-	}
-
-	var k3kService v1.Service
-	if err := client.Get(ctx, nn, &k3kService); err != nil {
-		return nil, err
-	}
-
-	if k3kService.Spec.Type == v1.ServiceTypeNodePort {
-		nodePort := k3kService.Spec.Ports[0].NodePort
-
-		restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig)
-		if err != nil {
-			return nil, err
+	if token != "" {
+		cluster.Spec.TokenSecretRef = &v1.SecretReference{
+			Name:      k3kcluster.TokenSecretName(name),
+			Namespace: namespace,
 		}
-		hostURL := fmt.Sprintf("https://%s:%d", serverIP, nodePort)
-		restConfig.Host = hostURL
-
-		clientConfig := generateKubeconfigFromRest(restConfig)
-
-		b, err := clientcmd.Write(clientConfig)
-		if err != nil {
-			return nil, err
-		}
-		kubeconfig = b
 	}
-
-	return kubeconfig, nil
-}
-
-func generateKubeconfigFromRest(config *rest.Config) clientcmdapi.Config {
-	clusters := make(map[string]*clientcmdapi.Cluster)
-	clusters["default-cluster"] = &clientcmdapi.Cluster{
-		Server:                   config.Host,
-		CertificateAuthorityData: config.CAData,
-	}
-
-	contexts := make(map[string]*clientcmdapi.Context)
-	contexts["default-context"] = &clientcmdapi.Context{
-		Cluster:   "default-cluster",
-		Namespace: "default",
-		AuthInfo:  "default",
-	}
-
-	authinfos := make(map[string]*clientcmdapi.AuthInfo)
-	authinfos["default"] = &clientcmdapi.AuthInfo{
-		ClientCertificateData: config.CertData,
-		ClientKeyData:         config.KeyData,
-	}
-
-	clientConfig := clientcmdapi.Config{
-		Kind:           "Config",
-		APIVersion:     "v1",
-		Clusters:       clusters,
-		Contexts:       contexts,
-		CurrentContext: "default-context",
-		AuthInfos:      authinfos,
-	}
-
-	return clientConfig
+	return cluster
 }

cli/cmds/cluster/delete.go

@@ -1 +1,48 @@
 package cluster
+
+import (
+	"context"
+
+	"github.com/rancher/k3k/cli/cmds"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var (
+	clusterDeleteFlags = []cli.Flag{
+		cli.StringFlag{
+			Name:        "name",
+			Usage:       "name of the cluster",
+			Destination: &name,
+		},
+	}
+)
+
+func delete(clx *cli.Context) error {
+	ctx := context.Background()
+
+	restConfig, err := clientcmd.BuildConfigFromFlags("", cmds.Kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	ctrlClient, err := client.New(restConfig, client.Options{
+		Scheme: Scheme,
+	})
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("deleting [%s] cluster", name)
+	cluster := v1alpha1.Cluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: cmds.Namespace(),
+		},
+	}
+	return ctrlClient.Delete(ctx, &cluster)
+}

cli/cmds/kubeconfig/kubeconfig.go

@@ -0,0 +1,163 @@
+package kubeconfig
+
+import (
+	"context"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/rancher/k3k/cli/cmds"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/certs"
+	"github.com/rancher/k3k/pkg/controller/kubeconfig"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apiserver/pkg/authentication/user"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func init() {
+	_ = clientgoscheme.AddToScheme(Scheme)
+	_ = v1alpha1.AddToScheme(Scheme)
+}
+
+var (
+	Scheme                  = runtime.NewScheme()
+	name                    string
+	cn                      string
+	org                     cli.StringSlice
+	altNames                cli.StringSlice
+	expirationDays          int64
+	configName              string
+	generateKubeconfigFlags = []cli.Flag{
+		cli.StringFlag{
+			Name:        "name",
+			Usage:       "cluster name",
+			Destination: &name,
+		},
+		cli.StringFlag{
+			Name:        "config-name",
+			Usage:       "the name of the generated kubeconfig file",
+			Destination: &configName,
+		},
+		cli.StringFlag{
+			Name:        "cn",
+			Usage:       "Common name (CN) of the generated certificates for the kubeconfig",
+			Destination: &cn,
+			Value:       controller.AdminCommonName,
+		},
+		cli.StringSliceFlag{
+			Name:  "org",
+			Usage: "Organization name (ORG) of the generated certificates for the kubeconfig",
+			Value: &org,
+		},
+		cli.StringSliceFlag{
+			Name:  "altNames",
+			Usage: "altNames of the generated certificates for the kubeconfig",
+			Value: &altNames,
+		},
+		cli.Int64Flag{
+			Name:        "expiration-days",
+			Usage:       "Expiration date of the certificates used for the kubeconfig",
+			Destination: &expirationDays,
+			Value:       356,
+		},
+	}
+)
+
+var subcommands = []cli.Command{
+	{
+		Name:            "generate",
+		Usage:           "Generate kubeconfig for clusters",
+		SkipFlagParsing: false,
+		SkipArgReorder:  true,
+		Action:          generate,
+		Flags:           append(cmds.CommonFlags, generateKubeconfigFlags...),
+	},
+}
+
+func NewCommand() cli.Command {
+	return cli.Command{
+		Name:        "kubeconfig",
+		Usage:       "Manage kubeconfig for clusters",
+		Subcommands: subcommands,
+	}
+}
+
+func generate(clx *cli.Context) error {
+	var cluster v1alpha1.Cluster
+	ctx := context.Background()
+
+	restConfig, err := clientcmd.BuildConfigFromFlags("", cmds.Kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	ctrlClient, err := client.New(restConfig, client.Options{
+		Scheme: Scheme,
+	})
+	if err != nil {
+		return err
+	}
+	clusterKey := types.NamespacedName{
+		Name:      name,
+		Namespace: cmds.Namespace(),
+	}
+
+	if err := ctrlClient.Get(ctx, clusterKey, &cluster); err != nil {
+		return err
+	}
+
+	url, err := url.Parse(restConfig.Host)
+	if err != nil {
+		return err
+	}
+	host := strings.Split(url.Host, ":")
+
+	certAltNames := certs.AddSANs(altNames)
+	if org == nil {
+		org = cli.StringSlice{user.SystemPrivilegedGroup}
+	}
+	cfg := kubeconfig.KubeConfig{
+		CN:         cn,
+		ORG:        org,
+		ExpiryDate: time.Hour * 24 * time.Duration(expirationDays),
+		AltNames:   certAltNames,
+	}
+	logrus.Infof("waiting for cluster to be available..")
+	var kubeconfig []byte
+	if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
+		kubeconfig, err = cfg.Extract(ctx, ctrlClient, &cluster, host[0])
+		if err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	pwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	if configName == "" {
+		configName = cluster.Name + "-kubeconfig.yaml"
+	}
+
+	logrus.Infof(`You can start using the cluster with: 
+
+	export KUBECONFIG=%s
+	kubectl cluster-info
+	`, filepath.Join(pwd, configName))
+
+	return os.WriteFile(configName, kubeconfig, 0644)
+}

cli/cmds/root.go

@@ -5,9 +5,14 @@ import (
 	"github.com/urfave/cli"
 )
 
+const (
+	defaultNamespace = "default"
+)
+
 var (
 	debug       bool
 	Kubeconfig  string
+	namespace   string
 	CommonFlags = []cli.Flag{
 		cli.StringFlag{
 			Name:        "kubeconfig",
@@ -15,6 +20,11 @@ var (
 			Usage:       "Kubeconfig path",
 			Destination: &Kubeconfig,
 		},
+		cli.StringFlag{
+			Name:        "namespace",
+			Usage:       "Namespace to create the k3k cluster in",
+			Destination: &namespace,
+		},
 	}
 )
 
@@ -40,3 +50,10 @@ func NewApp() *cli.App {
 
 	return app
 }
+
+func Namespace() string {
+	if namespace == "" {
+		return defaultNamespace
+	}
+	return namespace
+}

cli/main.go

@@ -5,12 +5,13 @@ import (
 
 	"github.com/rancher/k3k/cli/cmds"
 	"github.com/rancher/k3k/cli/cmds/cluster"
+	"github.com/rancher/k3k/cli/cmds/kubeconfig"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
 const (
-	program   = "k3k"
+	program   = "k3kcli"
 	version   = "dev"
 	gitCommit = "HEAD"
 )
@@ -18,7 +19,8 @@ const (
 func main() {
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
-		cluster.NewClusterCommand(),
+		cluster.NewCommand(),
+		kubeconfig.NewCommand(),
 	}
 	app.Version = version + " (" + gitCommit + ")"
 

examples/clusterset.yaml

@@ -0,0 +1,11 @@
+apiVersion: k3k.io/v1alpha1
+kind: ClusterSet
+metadata:
+  name: clusterset-example
+# spec:
+  # disableNetworkPolicy: false
+  # allowedNodeTypes:
+  # - "shared"
+  # - "virtual"
+  # podSecurityAdmissionLevel: "baseline"
+  # defaultPriorityClass: "lowpriority"

examples/multiple-servers.yaml

@@ -3,6 +3,7 @@ kind: Cluster
 metadata:
   name: example1
 spec:
+  mode: "shared"
   servers: 1
   agents: 3
   token: test

examples/single-server.yaml

@@ -3,6 +3,7 @@ kind: Cluster
 metadata:
   name: single-server
 spec:
+  mode: "shared"
   servers: 1
   agents: 3
   token: test

go.mod

@@ -1,89 +1,129 @@
 module github.com/rancher/k3k
 
-go 1.20
+go 1.22.0
+
+toolchain go1.22.7
 
 replace (
-	go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1
-	go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.9-k3s1
+	github.com/google/cel-go => github.com/google/cel-go v0.17.7
+	github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_model => github.com/prometheus/client_model v0.6.1
+	github.com/prometheus/common => github.com/prometheus/common v0.47.0
+	golang.org/x/term => golang.org/x/term v0.15.0
 )
 
 require (
-	github.com/sirupsen/logrus v1.8.1
+	github.com/go-logr/zapr v1.3.0
+	github.com/onsi/ginkgo/v2 v2.20.1
+	github.com/onsi/gomega v1.36.0
+	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_model v0.6.1
+	github.com/rancher/dynamiclistener v1.27.5
+	github.com/sirupsen/logrus v1.9.3
 	github.com/urfave/cli v1.22.12
-	go.etcd.io/etcd/api/v3 v3.5.9
-	go.etcd.io/etcd/client/v3 v3.5.5
-	k8s.io/api v0.26.1
-	k8s.io/apimachinery v0.26.1
-	k8s.io/client-go v0.26.1
-	k8s.io/klog v1.0.0
+	github.com/virtual-kubelet/virtual-kubelet v1.11.0
+	go.etcd.io/etcd/api/v3 v3.5.14
+	go.etcd.io/etcd/client/v3 v3.5.14
+	go.uber.org/zap v1.26.0
+	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/api v0.29.11
+	k8s.io/apimachinery v0.29.11
+	k8s.io/apiserver v0.29.11
+	k8s.io/client-go v0.29.11
+	k8s.io/component-base v0.29.11
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+	sigs.k8s.io/controller-runtime v0.17.5
 )
 
 require (
+	github.com/NYTimes/gziphandler v1.1.1 // indirect
+	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
-	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.20.0 // indirect
-	github.com/go-openapi/swag v0.19.14 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/uuid v1.1.2 // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
-	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
-	golang.org/x/sys v0.3.0 // indirect
-	golang.org/x/term v0.3.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
-	google.golang.org/grpc v1.49.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.26.0 // indirect
-	k8s.io/component-base v0.26.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
-)
-
-require (
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/cel-go v0.20.1 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/spdystream v0.4.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/rancher/dynamiclistener v0.3.5
-	golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect
-	golang.org/x/text v0.5.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/prometheus/client_golang v1.19.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
+	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/term v0.25.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
+	google.golang.org/grpc v1.65.0 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiserver v0.26.1
-	k8s.io/klog/v2 v2.80.1
-	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
-	sigs.k8s.io/controller-runtime v0.14.1
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.29.2 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kms v0.31.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )

go.sum

@@ -1,595 +1,339 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
+github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
+github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18=
+github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/bombsimon/logrusr/v3 v3.1.0 h1:zORbLM943D+hDMGgyjMhSAz/iDz86ZV72qaak/CA0zQ=
+github.com/bombsimon/logrusr/v3 v3.1.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
-github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
+github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
+github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
-github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
+github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
-github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
-github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
-github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
-github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
-github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
-github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
+github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
+github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
+github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
+github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
+github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
-github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ=
+github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
+github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
+github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA=
+github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
-github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
+github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1 h1:y4ont0HdnS7gtWNTXM8gahpKjAHtctgON/sjVRthlZY=
-github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k=
-github.com/k3s-io/etcd/client/v3 v3.5.9-k3s1 h1:Knr/8l7Sx92zUyevYO0gIO5P6EEc6ztvRO5EzSnMy+A=
-github.com/k3s-io/etcd/client/v3 v3.5.9-k3s1/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
-github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM=
-github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8=
+github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc=
-github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo=
+github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI=
+github.com/onsi/gomega v1.36.0 h1:Pb12RlruUtj4XUuPUqeEWc6j5DkVVVA49Uf6YLfC95Y=
+github.com/onsi/gomega v1.36.0/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
-github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
-github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
-github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
-github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
-github.com/rancher/dynamiclistener v0.3.5 h1:5TaIHvkDGmZKvc96Huur16zfTKOiLhDtK4S+WV0JA6A=
-github.com/rancher/dynamiclistener v0.3.5/go.mod h1:dW/YF6/m2+uEyJ5VtEcd9THxda599HP6N9dSXk81+k0=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.47.0 h1:p5Cz0FNHo7SnWOmWmoRozVcjEp0bIVU8cV7OShpjL1k=
+github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/rancher/dynamiclistener v1.27.5 h1:FA/s9vbQzGz1Au3BuFvdbBfBBUmHGXGR3xoliwR4qfY=
+github.com/rancher/dynamiclistener v1.27.5/go.mod h1:VqBaJNi+bZmre0+gi+2Jb6jbn7ovHzRueW+M7QhVKsk=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
-github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
+github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
 github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8=
 github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/virtual-kubelet/virtual-kubelet v1.11.0 h1:LOMcZQfP083xmYH9mYtyHAR+ybFbK1uMaRA+EtDcd1I=
+github.com/virtual-kubelet/virtual-kubelet v1.11.0/go.mod h1:WQfPHbIlzfhMNYkh6hFXF1ctGfNM8UJCYLYpLa/trxc=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE=
-go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
-go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
-go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
+go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/etcd/api/v3 v3.5.14 h1:vHObSCxyB9zlF60w7qzAdTcGaglbJOpSj1Xj9+WGxq0=
+go.etcd.io/etcd/api/v3 v3.5.14/go.mod h1:BmtWcRlQvwa1h3G2jvKYwIQy4PkHlDej5t7uLMUdJUU=
+go.etcd.io/etcd/client/pkg/v3 v3.5.14 h1:SaNH6Y+rVEdxfpA2Jr5wkEvN6Zykme5+YnbCkxvuWxQ=
+go.etcd.io/etcd/client/pkg/v3 v3.5.14/go.mod h1:8uMgAokyG1czCtIdsq+AGyYQMvpIKnSvPjFMunkgeZI=
+go.etcd.io/etcd/client/v2 v2.305.10 h1:MrmRktzv/XF8CvtQt+P6wLUlURaNpSDJHFZhe//2QE4=
+go.etcd.io/etcd/client/v2 v2.305.10/go.mod h1:m3CKZi69HzilhVqtPDcjhSGp+kA1OmbNn0qamH80xjA=
+go.etcd.io/etcd/client/v3 v3.5.14 h1:CWfRs4FDaDoSz81giL7zPpZH2Z35tbOrAJkkjMqOupg=
+go.etcd.io/etcd/client/v3 v3.5.14/go.mod h1:k3XfdV/VIHy/97rqWjoUzrj9tk7GgJGH9J8L4dNXmAk=
+go.etcd.io/etcd/pkg/v3 v3.5.10 h1:WPR8K0e9kWl1gAhB5A7gEa5ZBTNkT9NdNWrR8Qpo1CM=
+go.etcd.io/etcd/pkg/v3 v3.5.10/go.mod h1:TKTuCKKcF1zxmfKWDkfz5qqYaE3JncKKZPFf8c1nFUs=
+go.etcd.io/etcd/raft/v3 v3.5.10 h1:cgNAYe7xrsrn/5kXMSaH8kM/Ky8mAdMqGOxyYwpP0LA=
+go.etcd.io/etcd/raft/v3 v3.5.10/go.mod h1:odD6kr8XQXTy9oQnyMPBOr0TVe+gT0neQhElQ6jbGRc=
+go.etcd.io/etcd/server/v3 v3.5.10 h1:4NOGyOwD5sUZ22PiWYKmfxqoeh72z6EhYjNosKGLmZg=
+go.etcd.io/etcd/server/v3 v3.5.10/go.mod h1:gBplPHfs6YI0L+RpGkTQO7buDbHv5HJGG/Bst0/zIPo=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=
+go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
+go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ=
+go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
+go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
+go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
+go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
+go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
+go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
+go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
+go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 h1:Frnccbp+ok2GkUS2tC84yAq/U9Vg+0sIO7aRL3T4Xnc=
-golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg=
-golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
+golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
-gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
+gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I=
-google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw=
-google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
+google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -598,71 +342,57 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ=
-k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg=
-k8s.io/apiextensions-apiserver v0.26.0 h1:Gy93Xo1eg2ZIkNX/8vy5xviVSxwQulsnUdQ00nEdpDo=
-k8s.io/apiextensions-apiserver v0.26.0/go.mod h1:7ez0LTiyW5nq3vADtK6C3kMESxadD51Bh6uz3JOlqWQ=
-k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ=
-k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
-k8s.io/apiserver v0.26.1 h1:6vmnAqCDO194SVCPU3MU8NcDgSqsUA62tBUSWrFXhsc=
-k8s.io/apiserver v0.26.1/go.mod h1:wr75z634Cv+sifswE9HlAo5FQ7UoUauIICRlOE+5dCg=
-k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU=
-k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE=
-k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4=
-k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU=
-k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
-k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
-k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
-k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
-k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
-k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y=
-k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/controller-runtime v0.14.1 h1:vThDes9pzg0Y+UbCPY3Wj34CGIYPgdmspPm2GIpxpzM=
-sigs.k8s.io/controller-runtime v0.14.1/go.mod h1:GaRkrY8a7UZF0kqFFbUKG7n9ICiTY5T55P1RiE3UZlU=
-sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
-sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+k8s.io/api v0.29.11 h1:6FwDo33f1WX5Yu0RQTX9YAd3wth8Ik0B4SXQKsoQfbk=
+k8s.io/api v0.29.11/go.mod h1:3TDAW1OpFbz/Yx5r0W06b6eiAfHEwtH61VYDzpTU4Ng=
+k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg=
+k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8=
+k8s.io/apimachinery v0.29.11 h1:55+6ue9advpA7T0sX2ZJDHCLKuiFfrAAR/39VQN9KEQ=
+k8s.io/apimachinery v0.29.11/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y=
+k8s.io/apiserver v0.29.11 h1:EXcv4/3iIKWG5tWI2ywdMY86jpxYw6WDAdMrBKUMkSc=
+k8s.io/apiserver v0.29.11/go.mod h1:lnoWXh0J75ei1h4/F5ZbOEd7byiAVasherLB6Snzl/I=
+k8s.io/client-go v0.29.11 h1:mBX7Ub0uqpLMwWz3J/AGS/xKOZsjr349qZ1vxVoL1l8=
+k8s.io/client-go v0.29.11/go.mod h1:WOEoi/eLg2YEg3/yEd7YK3CNScYkM8AEScQadxUnaTE=
+k8s.io/component-base v0.29.11 h1:H3GJIyDNPrscvXGP6wx+9gApcwwmrUd0YtCGp5BcHBA=
+k8s.io/component-base v0.29.11/go.mod h1:0qu1WStER4wu5o8RMRndZUWPVcPH1XBy/QQiDcD6lew=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kms v0.31.0 h1:KchILPfB1ZE+ka7223mpU5zeFNkmb45jl7RHnlImUaI=
+k8s.io/kms v0.31.0/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
+k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
+k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
+sigs.k8s.io/controller-runtime v0.17.5 h1:1FI9Lm7NiOOmBsgTV36/s2XrEFXnO2C4sbg/Zme72Rw=
+sigs.k8s.io/controller-runtime v0.17.5/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

hack/update-codegen.sh

@@ -4,18 +4,25 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+set -x
 CODEGEN_GIT_PKG=https://github.com/kubernetes/code-generator.git
 git clone --depth 1 ${CODEGEN_GIT_PKG} || true
 
+K8S_VERSION=$(cat go.mod | grep -m1 "k8s.io/apiserver" | cut -d " " -f 2)
 SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 CODEGEN_PKG=./code-generator
 
-"${CODEGEN_PKG}/generate-groups.sh" \
- "deepcopy" \
- github.com/rancher/k3k/pkg/generated \
- github.com/rancher/k3k/pkg/apis \
- "k3k.io:v1alpha1" \
- --go-header-file "${SCRIPT_ROOT}"/hack/boilerplate.go.txt \
- --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.."
+# cd into the git dir to checkout the code gen version compatible with the k8s version that this is using
+cd $CODEGEN_PKG
+git fetch origin tag ${K8S_VERSION}
+git checkout ${K8S_VERSION}
+cd -
+
+source ${CODEGEN_PKG}/kube_codegen.sh
+
+kube::codegen::gen_helpers \
+    --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt" \
+    --input-pkg-root "${SCRIPT_ROOT}/pkg/apis" \
+    --output-base "${SCRIPT_ROOT}/pkg/apis"
 
 rm -rf code-generator

k3k-kubelet/README.md

@@ -0,0 +1,34 @@
+## Virtual Kubelet
+
+This package provides an impelementation of a virtual cluster node using [virtual-kubelet](https://github.com/virtual-kubelet/virtual-kubelet).
+
+The implementation is based on several projects, including:
+- [Virtual Kubelet](https://github.com/virtual-kubelet/virtual-kubelet)
+- [Kubectl](https://github.com/kubernetes/kubectl)
+- [Client-go](https://github.com/kubernetes/client-go)
+- [Azure-Aci](https://github.com/virtual-kubelet/azure-aci)
+
+## Overview
+
+This project creates a node that registers itself in the virtual cluster. When workloads are scheduled to this node, it simply creates/updates the workload on the host cluster.  
+
+## Usage
+
+Build/Push the image using (from the root of rancher/k3k):
+
+```
+make build
+docker buildx build -f package/Dockerfile . -t $REPO/$IMAGE:$TAG
+```
+
+When running, it is recommended to deploy a k3k cluster with 1 server (with `--disable-agent` as a server arg) and no agents (so that the workloads can only be scheduled on the virtual node/host cluster).
+
+After the image is built, it should be deployed with the following ENV vars set:
+- `CLUSTER_NAME` should be the name of the cluster.
+- `CLUSTER_NAMESPACE` should be the namespace the cluster is running in.
+- `HOST_KUBECONFIG` should be the path on the local filesystem (in container) to a kubeconfig for the host cluster (likely stored in a secret/mounted as a volume).
+- `VIRT_KUBECONFIG`should be the path on the local filesystem (in container) to a kubeconfig for the virtual cluster (likely stored in a secret/mounted as a volume).
+- `VIRT_POD_IP` should be the IP that the container is accessible from.
+
+This project is still under development and there are many features yet to be implemented, but it can run a basic nginx pod.
+

k3k-kubelet/config.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"errors"
+	"os"
+
+	"gopkg.in/yaml.v2"
+)
+
+// config has all virtual-kubelet startup options
+type config struct {
+	ClusterName       string `yaml:"clusterName,omitempty"`
+	ClusterNamespace  string `yaml:"clusterNamespace,omitempty"`
+	NodeName          string `yaml:"nodeName,omitempty"`
+	Token             string `yaml:"token,omitempty"`
+	AgentHostname     string `yaml:"agentHostname,omitempty"`
+	HostConfigPath    string `yaml:"hostConfigPath,omitempty"`
+	VirtualConfigPath string `yaml:"virtualConfigPath,omitempty"`
+	KubeletPort       string `yaml:"kubeletPort,omitempty"`
+	ServerIP          string `yaml:"serverIP,omitempty"`
+}
+
+func (c *config) unmarshalYAML(data []byte) error {
+	var conf config
+
+	if err := yaml.Unmarshal(data, &conf); err != nil {
+		return err
+	}
+
+	if c.ClusterName == "" {
+		c.ClusterName = conf.ClusterName
+	}
+	if c.ClusterNamespace == "" {
+		c.ClusterNamespace = conf.ClusterNamespace
+	}
+	if c.HostConfigPath == "" {
+		c.HostConfigPath = conf.HostConfigPath
+	}
+	if c.VirtualConfigPath == "" {
+		c.VirtualConfigPath = conf.VirtualConfigPath
+	}
+	if c.KubeletPort == "" {
+		c.KubeletPort = conf.KubeletPort
+	}
+	if c.AgentHostname == "" {
+		c.AgentHostname = conf.AgentHostname
+	}
+	if c.NodeName == "" {
+		c.NodeName = conf.NodeName
+	}
+	if c.Token == "" {
+		c.Token = conf.Token
+	}
+	if c.ServerIP == "" {
+		c.ServerIP = conf.ServerIP
+	}
+	return nil
+}
+
+func (c *config) validate() error {
+	if c.ClusterName == "" {
+		return errors.New("cluster name is not provided")
+	}
+	if c.ClusterNamespace == "" {
+		return errors.New("cluster namespace is not provided")
+	}
+	if c.AgentHostname == "" {
+		return errors.New("agent Hostname is not provided")
+	}
+	return nil
+}
+
+func (c *config) parse(path string) error {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil
+	}
+
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	return c.unmarshalYAML(b)
+}

k3k-kubelet/controller/configmap.go

@@ -0,0 +1,166 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/rancher/k3k/pkg/controller"
+	k3klog "github.com/rancher/k3k/pkg/log"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/util/retry"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+type ConfigMapSyncer struct {
+	mutex sync.RWMutex
+	// VirtualClient is the client for the virtual cluster
+	VirtualClient client.Client
+	// CoreClient is the client for the host cluster
+	HostClient client.Client
+	// TranslateFunc is the function that translates a given resource from it's virtual representation to the host
+	// representation
+	TranslateFunc func(*corev1.ConfigMap) (*corev1.ConfigMap, error)
+	// Logger is the logger that the controller will use
+	Logger *k3klog.Logger
+	// objs are the objects that the syncer should watch/syncronize. Should only be manipulated
+	// through add/remove
+	objs sets.Set[types.NamespacedName]
+}
+
+// Reconcile implements reconcile.Reconciler and synchronizes the objects in objs to the host cluster
+func (c *ConfigMapSyncer) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	if !c.isWatching(req.NamespacedName) {
+		// return immediately without re-enqueueing. We aren't watching this resource
+		return reconcile.Result{}, nil
+	}
+	var virtual corev1.ConfigMap
+
+	if err := c.VirtualClient.Get(ctx, req.NamespacedName, &virtual); err != nil {
+		return reconcile.Result{
+			Requeue: true,
+		}, fmt.Errorf("unable to get configmap %s/%s from virtual cluster: %w", req.Namespace, req.Name, err)
+	}
+	translated, err := c.TranslateFunc(&virtual)
+	if err != nil {
+		return reconcile.Result{
+			Requeue: true,
+		}, fmt.Errorf("unable to translate configmap %s/%s from virtual cluster: %w", req.Namespace, req.Name, err)
+	}
+	translatedKey := types.NamespacedName{
+		Namespace: translated.Namespace,
+		Name:      translated.Name,
+	}
+	var host corev1.ConfigMap
+	if err = c.HostClient.Get(ctx, translatedKey, &host); err != nil {
+		if apierrors.IsNotFound(err) {
+			err = c.HostClient.Create(ctx, translated)
+			// for simplicity's sake, we don't check for conflict errors. The existing object will get
+			// picked up on in the next re-enqueue
+			return reconcile.Result{
+					Requeue: true,
+				}, fmt.Errorf("unable to create host configmap %s/%s for virtual configmap %s/%s: %w",
+					translated.Namespace, translated.Name, req.Namespace, req.Name, err)
+		}
+		return reconcile.Result{Requeue: true}, fmt.Errorf("unable to get host configmap %s/%s: %w", translated.Namespace, translated.Name, err)
+	}
+	// we are going to use the host in order to avoid conflicts on update
+	host.Data = translated.Data
+	if host.Labels == nil {
+		host.Labels = make(map[string]string, len(translated.Labels))
+	}
+	// we don't want to override labels made on the host cluster by other applications
+	// but we do need to make sure the labels that the kubelet uses to track host cluster values
+	// are being tracked appropriately
+	for key, value := range translated.Labels {
+		host.Labels[key] = value
+	}
+	if err = c.HostClient.Update(ctx, &host); err != nil {
+		return reconcile.Result{
+				Requeue: true,
+			}, fmt.Errorf("unable to update host configmap %s/%s for virtual configmap %s/%s: %w",
+				translated.Namespace, translated.Name, req.Namespace, req.Name, err)
+
+	}
+	return reconcile.Result{}, nil
+}
+
+// isWatching is a utility method to determine if a key is in objs without the caller needing
+// to handle mutex lock/unlock.
+func (c *ConfigMapSyncer) isWatching(key types.NamespacedName) bool {
+	c.mutex.RLock()
+	defer c.mutex.RUnlock()
+	return c.objs.Has(key)
+}
+
+// AddResource adds a given resource to the list of resources that will be synced. Safe to call multiple times for the
+// same resource.
+func (c *ConfigMapSyncer) AddResource(ctx context.Context, namespace, name string) error {
+	objKey := types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}
+	// if we already sync this object, no need to writelock/add it
+	if c.isWatching(objKey) {
+		return nil
+	}
+	// lock in write mode since we are now adding the key
+	c.mutex.Lock()
+	if c.objs == nil {
+		c.objs = sets.Set[types.NamespacedName]{}
+	}
+	c.objs = c.objs.Insert(objKey)
+	c.mutex.Unlock()
+	_, err := c.Reconcile(ctx, reconcile.Request{
+		NamespacedName: objKey,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to reconcile new object %s/%s: %w", objKey.Namespace, objKey.Name, err)
+	}
+	return nil
+}
+
+// RemoveResource removes a given resource from the list of resources that will be synced. Safe to call for an already
+// removed resource.
+func (c *ConfigMapSyncer) RemoveResource(ctx context.Context, namespace, name string) error {
+	objKey := types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}
+	// if we don't sync this object, no need to writelock/add it
+	if !c.isWatching(objKey) {
+		return nil
+	}
+
+	if err := retry.OnError(controller.Backoff, func(err error) bool {
+		return err != nil
+	}, func() error {
+		return c.removeHostConfigMap(ctx, namespace, name)
+	}); err != nil {
+		return fmt.Errorf("unable to remove configmap: %w", err)
+	}
+	c.mutex.Lock()
+	if c.objs == nil {
+		c.objs = sets.Set[types.NamespacedName]{}
+	}
+	c.objs = c.objs.Delete(objKey)
+	c.mutex.Unlock()
+	return nil
+}
+
+func (c *ConfigMapSyncer) removeHostConfigMap(ctx context.Context, virtualNamespace, virtualName string) error {
+	var vConfigMap corev1.ConfigMap
+	err := c.VirtualClient.Get(ctx, types.NamespacedName{Namespace: virtualNamespace, Name: virtualName}, &vConfigMap)
+	if err != nil {
+		return fmt.Errorf("unable to get virtual configmap %s/%s: %w", virtualNamespace, virtualName, err)
+	}
+	translated, err := c.TranslateFunc(&vConfigMap)
+	if err != nil {
+		return fmt.Errorf("unable to translate virtual secret: %s/%s: %w", virtualNamespace, virtualName, err)
+	}
+	return c.HostClient.Delete(ctx, translated)
+}

k3k-kubelet/controller/handler.go

@@ -0,0 +1,115 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/rancher/k3k/k3k-kubelet/translate"
+	k3klog "github.com/rancher/k3k/pkg/log"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+type ControllerHandler struct {
+	sync.RWMutex
+	// Mgr is the manager used to run new controllers - from the virtual cluster
+	Mgr manager.Manager
+	// Scheme is the scheme used to run new controllers - from the virtual cluster
+	Scheme runtime.Scheme
+	// HostClient is the client used to communicate with the host cluster
+	HostClient client.Client
+	// VirtualClient is the client used to communicate with the virtual cluster
+	VirtualClient client.Client
+	// Translater is the translater that will be used to adjust objects before they
+	// are made on the host cluster
+	Translater translate.ToHostTranslater
+	// Logger is the logger that the controller will use to log errors
+	Logger *k3klog.Logger
+	// controllers are the controllers which are currently running
+	controllers map[schema.GroupVersionKind]updateableReconciler
+}
+
+// updateableReconciler is a reconciler that only syncs specific resources (by name/namespace). This list can
+// be altered through the Add and Remove methods
+type updateableReconciler interface {
+	reconcile.Reconciler
+	AddResource(ctx context.Context, namespace string, name string) error
+	RemoveResource(ctx context.Context, namespace string, name string) error
+}
+
+func (c *ControllerHandler) AddResource(ctx context.Context, obj client.Object) error {
+	c.RLock()
+	controllers := c.controllers
+	if controllers != nil {
+		if r, ok := c.controllers[obj.GetObjectKind().GroupVersionKind()]; ok {
+			err := r.AddResource(ctx, obj.GetNamespace(), obj.GetName())
+			c.RUnlock()
+			return err
+		}
+	}
+	// we need to manually lock/unlock since we intned on write locking to add a new controller
+	c.RUnlock()
+	var r updateableReconciler
+	switch obj.(type) {
+	case *v1.Secret:
+		r = &SecretSyncer{
+			HostClient:    c.HostClient,
+			VirtualClient: c.VirtualClient,
+			// TODO: Need actual function
+			TranslateFunc: func(s *v1.Secret) (*v1.Secret, error) {
+				// note that this doesn't do any type safety - fix this
+				// when generics work
+				c.Translater.TranslateTo(s)
+				return s, nil
+			},
+			Logger: c.Logger,
+		}
+	case *v1.ConfigMap:
+		r = &ConfigMapSyncer{
+			HostClient:    c.HostClient,
+			VirtualClient: c.VirtualClient,
+			// TODO: Need actual function
+			TranslateFunc: func(s *v1.ConfigMap) (*v1.ConfigMap, error) {
+				c.Translater.TranslateTo(s)
+				return s, nil
+			},
+			Logger: c.Logger,
+		}
+	default:
+		// TODO: Technically, the configmap/secret syncers are relatively generic, and this
+		// logic could be used for other types.
+		return fmt.Errorf("unrecognized type: %T", obj)
+
+	}
+	err := ctrl.NewControllerManagedBy(c.Mgr).
+		For(&v1.ConfigMap{}).
+		Complete(r)
+	if err != nil {
+		return fmt.Errorf("unable to start configmap controller: %w", err)
+	}
+	c.Lock()
+	if c.controllers == nil {
+		c.controllers = map[schema.GroupVersionKind]updateableReconciler{}
+	}
+	c.controllers[obj.GetObjectKind().GroupVersionKind()] = r
+	c.Unlock()
+
+	return r.AddResource(ctx, obj.GetNamespace(), obj.GetName())
+}
+
+func (c *ControllerHandler) RemoveResource(ctx context.Context, obj client.Object) error {
+	// since we aren't adding a new controller, we don't need to lock
+	c.RLock()
+	ctrl, ok := c.controllers[obj.GetObjectKind().GroupVersionKind()]
+	c.RUnlock()
+	if !ok {
+		return fmt.Errorf("no controller found for gvk" + obj.GetObjectKind().GroupVersionKind().String())
+	}
+	return ctrl.RemoveResource(ctx, obj.GetNamespace(), obj.GetName())
+}

k3k-kubelet/controller/persistentvolumeclaims.go

@@ -0,0 +1,121 @@
+package controller
+
+import (
+	"context"
+
+	"github.com/rancher/k3k/k3k-kubelet/translate"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/log"
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+	pvcController    = "pvc-syncer-controller"
+	pvcFinalizerName = "pvc.k3k.io/finalizer"
+)
+
+type PVCReconciler struct {
+	virtualClient    ctrlruntimeclient.Client
+	hostClient       ctrlruntimeclient.Client
+	clusterName      string
+	clusterNamespace string
+	Scheme           *runtime.Scheme
+	HostScheme       *runtime.Scheme
+	logger           *log.Logger
+	Translater       translate.ToHostTranslater
+}
+
+// AddPVCSyncer adds persistentvolumeclaims syncer controller to k3k-kubelet
+func AddPVCSyncer(ctx context.Context, virtMgr, hostMgr manager.Manager, clusterName, clusterNamespace string, logger *log.Logger) error {
+	translater := translate.ToHostTranslater{
+		ClusterName:      clusterName,
+		ClusterNamespace: clusterNamespace,
+	}
+	// initialize a new Reconciler
+	reconciler := PVCReconciler{
+		virtualClient:    virtMgr.GetClient(),
+		hostClient:       hostMgr.GetClient(),
+		Scheme:           virtMgr.GetScheme(),
+		HostScheme:       hostMgr.GetScheme(),
+		logger:           logger.Named(pvcController),
+		Translater:       translater,
+		clusterName:      clusterName,
+		clusterNamespace: clusterNamespace,
+	}
+	return ctrl.NewControllerManagedBy(virtMgr).
+		For(&v1.PersistentVolumeClaim{}).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: maxConcurrentReconciles,
+		}).
+		Complete(&reconciler)
+}
+
+func (r *PVCReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	log := r.logger.With("Cluster", r.clusterName, "PersistentVolumeClaim", req.NamespacedName)
+	var (
+		virtPVC v1.PersistentVolumeClaim
+		hostPVC v1.PersistentVolumeClaim
+		cluster v1alpha1.Cluster
+	)
+	if err := r.hostClient.Get(ctx, types.NamespacedName{Name: r.clusterName, Namespace: r.clusterNamespace}, &cluster); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	// handling persistent volume sync
+	if err := r.virtualClient.Get(ctx, req.NamespacedName, &virtPVC); err != nil {
+		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
+	}
+	syncedPVC := r.pvc(&virtPVC)
+	if err := controllerutil.SetControllerReference(&cluster, syncedPVC, r.HostScheme); err != nil {
+		return reconcile.Result{}, err
+	}
+	// handle deletion
+	if !virtPVC.DeletionTimestamp.IsZero() {
+		// deleting the synced service if exists
+		if err := r.hostClient.Delete(ctx, syncedPVC); !apierrors.IsNotFound(err) {
+			return reconcile.Result{}, err
+		}
+		// remove the finalizer after cleaning up the synced service
+		if controllerutil.RemoveFinalizer(&virtPVC, pvcFinalizerName) {
+			if err := r.virtualClient.Update(ctx, &virtPVC); err != nil {
+				return reconcile.Result{}, err
+			}
+		}
+		return reconcile.Result{}, nil
+	}
+
+	// getting the cluster for setting the controller reference
+
+	// Add finalizer if it does not exist
+	if controllerutil.AddFinalizer(&virtPVC, pvcFinalizerName) {
+		if err := r.virtualClient.Update(ctx, &virtPVC); err != nil {
+			return reconcile.Result{}, err
+		}
+	}
+	// create or update the pvc on host
+	if err := r.hostClient.Get(ctx, types.NamespacedName{Name: syncedPVC.Name, Namespace: r.clusterNamespace}, &hostPVC); err != nil {
+		if apierrors.IsNotFound(err) {
+			log.Info("creating the persistent volume for the first time on the host cluster")
+			return reconcile.Result{}, r.hostClient.Create(ctx, syncedPVC)
+		}
+		return reconcile.Result{}, err
+	}
+	log.Info("updating pvc on the host cluster")
+	return reconcile.Result{}, r.hostClient.Update(ctx, syncedPVC)
+
+}
+
+func (r *PVCReconciler) pvc(obj *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim {
+	hostPVC := obj.DeepCopy()
+	r.Translater.TranslateTo(hostPVC)
+	return hostPVC
+}

k3k-kubelet/controller/secret.go

@@ -0,0 +1,170 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/rancher/k3k/pkg/controller"
+	k3klog "github.com/rancher/k3k/pkg/log"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/util/retry"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+type SecretSyncer struct {
+	mutex sync.RWMutex
+	// VirtualClient is the client for the virtual cluster
+	VirtualClient client.Client
+	// CoreClient is the client for the host cluster
+	HostClient client.Client
+	// TranslateFunc is the function that translates a given resource from it's virtual representation to the host
+	// representation
+	TranslateFunc func(*corev1.Secret) (*corev1.Secret, error)
+	// Logger is the logger that the controller will use
+	Logger *k3klog.Logger
+	// objs are the objects that the syncer should watch/syncronize. Should only be manipulated
+	// through add/remove
+	objs sets.Set[types.NamespacedName]
+}
+
+// Reconcile implements reconcile.Reconciler and synchronizes the objects in objs to the host cluster
+func (s *SecretSyncer) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	if !s.isWatching(req.NamespacedName) {
+		// return immediately without re-enqueueing. We aren't watching this resource
+		return reconcile.Result{}, nil
+	}
+	var virtual corev1.Secret
+
+	if err := s.VirtualClient.Get(ctx, req.NamespacedName, &virtual); err != nil {
+		return reconcile.Result{
+			Requeue: true,
+		}, fmt.Errorf("unable to get secret %s/%s from virtual cluster: %w", req.Namespace, req.Name, err)
+	}
+	translated, err := s.TranslateFunc(&virtual)
+	if err != nil {
+		return reconcile.Result{
+			Requeue: true,
+		}, fmt.Errorf("unable to translate secret %s/%s from virtual cluster: %w", req.Namespace, req.Name, err)
+	}
+	translatedKey := types.NamespacedName{
+		Namespace: translated.Namespace,
+		Name:      translated.Name,
+	}
+	var host corev1.Secret
+	if err = s.HostClient.Get(ctx, translatedKey, &host); err != nil {
+		if apierrors.IsNotFound(err) {
+			err = s.HostClient.Create(ctx, translated)
+			// for simplicity's sake, we don't check for conflict errors. The existing object will get
+			// picked up on in the next re-enqueue
+			return reconcile.Result{
+					Requeue: true,
+				}, fmt.Errorf("unable to create host secret %s/%s for virtual secret %s/%s: %w",
+					translated.Namespace, translated.Name, req.Namespace, req.Name, err)
+		}
+		return reconcile.Result{Requeue: true}, fmt.Errorf("unable to get host secret %s/%s: %w", translated.Namespace, translated.Name, err)
+	}
+	// we are going to use the host in order to avoid conflicts on update
+	host.Data = translated.Data
+	if host.Labels == nil {
+		host.Labels = make(map[string]string, len(translated.Labels))
+	}
+	// we don't want to override labels made on the host cluster by other applications
+	// but we do need to make sure the labels that the kubelet uses to track host cluster values
+	// are being tracked appropriately
+	for key, value := range translated.Labels {
+		host.Labels[key] = value
+	}
+	if err = s.HostClient.Update(ctx, &host); err != nil {
+		return reconcile.Result{
+				Requeue: true,
+			}, fmt.Errorf("unable to update host secret %s/%s for virtual secret %s/%s: %w",
+				translated.Namespace, translated.Name, req.Namespace, req.Name, err)
+
+	}
+	return reconcile.Result{}, nil
+}
+
+// isWatching is a utility method to determine if a key is in objs without the caller needing
+// to handle mutex lock/unlock.
+func (s *SecretSyncer) isWatching(key types.NamespacedName) bool {
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
+	return s.objs.Has(key)
+}
+
+// AddResource adds a given resource to the list of resources that will be synced. Safe to call multiple times for the
+// same resource.
+func (s *SecretSyncer) AddResource(ctx context.Context, namespace, name string) error {
+	objKey := types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}
+	// if we already sync this object, no need to writelock/add it
+	if s.isWatching(objKey) {
+		return nil
+	}
+	// lock in write mode since we are now adding the key
+	s.mutex.Lock()
+	if s.objs == nil {
+		s.objs = sets.Set[types.NamespacedName]{}
+	}
+	s.objs = s.objs.Insert(objKey)
+	s.mutex.Unlock()
+	_, err := s.Reconcile(ctx, reconcile.Request{
+		NamespacedName: objKey,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to reconcile new object %s/%s: %w", objKey.Namespace, objKey.Name, err)
+	}
+	return nil
+}
+
+// RemoveResource removes a given resource from the list of resources that will be synced. Safe to call for an already
+// removed resource.
+func (s *SecretSyncer) RemoveResource(ctx context.Context, namespace, name string) error {
+	objKey := types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}
+	// if we don't sync this object, no need to writelock/add it
+	if !s.isWatching(objKey) {
+		return nil
+	}
+	// lock in write mode since we are now adding the key
+	if err := retry.OnError(controller.Backoff, func(err error) bool {
+		return err != nil
+	}, func() error {
+		return s.removeHostSecret(ctx, namespace, name)
+	}); err != nil {
+		return fmt.Errorf("unable to remove secret: %w", err)
+	}
+
+	s.mutex.Lock()
+	if s.objs == nil {
+		s.objs = sets.Set[types.NamespacedName]{}
+	}
+	s.objs = s.objs.Delete(objKey)
+	s.mutex.Unlock()
+	return nil
+}
+
+func (s *SecretSyncer) removeHostSecret(ctx context.Context, virtualNamespace, virtualName string) error {
+	var vSecret corev1.Secret
+	err := s.VirtualClient.Get(ctx, types.NamespacedName{
+		Namespace: virtualNamespace,
+		Name:      virtualName,
+	}, &vSecret)
+	if err != nil {
+		return fmt.Errorf("unable to get virtual secret %s/%s: %w", virtualNamespace, virtualName, err)
+	}
+	translated, err := s.TranslateFunc(&vSecret)
+	if err != nil {
+		return fmt.Errorf("unable to translate virtual secret: %s/%s: %w", virtualNamespace, virtualName, err)
+	}
+	return s.HostClient.Delete(ctx, translated)
+}

k3k-kubelet/controller/service.go

@@ -0,0 +1,126 @@
+package controller
+
+import (
+	"context"
+
+	"github.com/rancher/k3k/k3k-kubelet/translate"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/log"
+
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+	serviceSyncerController = "service-syncer-controller"
+	maxConcurrentReconciles = 1
+	serviceFinalizerName    = "service.k3k.io/finalizer"
+)
+
+type ServiceReconciler struct {
+	virtualClient    ctrlruntimeclient.Client
+	hostClient       ctrlruntimeclient.Client
+	clusterName      string
+	clusterNamespace string
+	Scheme           *runtime.Scheme
+	HostScheme       *runtime.Scheme
+	logger           *log.Logger
+	Translater       translate.ToHostTranslater
+}
+
+// AddServiceSyncer adds service syncer controller to the manager of the virtual cluster
+func AddServiceSyncer(ctx context.Context, virtMgr, hostMgr manager.Manager, clusterName, clusterNamespace string, logger *log.Logger) error {
+	translater := translate.ToHostTranslater{
+		ClusterName:      clusterName,
+		ClusterNamespace: clusterNamespace,
+	}
+	// initialize a new Reconciler
+	reconciler := ServiceReconciler{
+		virtualClient:    virtMgr.GetClient(),
+		hostClient:       hostMgr.GetClient(),
+		Scheme:           virtMgr.GetScheme(),
+		HostScheme:       hostMgr.GetScheme(),
+		logger:           logger.Named(serviceSyncerController),
+		Translater:       translater,
+		clusterName:      clusterName,
+		clusterNamespace: clusterNamespace,
+	}
+	return ctrl.NewControllerManagedBy(virtMgr).
+		For(&v1.Service{}).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: maxConcurrentReconciles,
+		}).
+		Complete(&reconciler)
+}
+
+func (s *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	log := s.logger.With("Cluster", s.clusterName, "Service", req.NamespacedName)
+	if req.Name == "kubernetes" || req.Name == "kube-dns" {
+		return reconcile.Result{}, nil
+	}
+	var (
+		virtService v1.Service
+		hostService v1.Service
+		cluster     v1alpha1.Cluster
+	)
+	// getting the cluster for setting the controller reference
+	if err := s.hostClient.Get(ctx, types.NamespacedName{Name: s.clusterName, Namespace: s.clusterNamespace}, &cluster); err != nil {
+		return reconcile.Result{}, err
+	}
+	if err := s.virtualClient.Get(ctx, req.NamespacedName, &virtService); err != nil {
+		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
+	}
+	syncedService := s.service(&virtService)
+	if err := controllerutil.SetControllerReference(&cluster, syncedService, s.HostScheme); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	// handle deletion
+	if !virtService.DeletionTimestamp.IsZero() {
+		// deleting the synced service if exists
+		if err := s.hostClient.Delete(ctx, syncedService); err != nil {
+			return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
+		}
+		// remove the finalizer after cleaning up the synced service
+		if controllerutil.ContainsFinalizer(&virtService, serviceFinalizerName) {
+			controllerutil.RemoveFinalizer(&virtService, serviceFinalizerName)
+			if err := s.virtualClient.Update(ctx, &virtService); err != nil {
+				return reconcile.Result{}, err
+			}
+		}
+		return reconcile.Result{}, nil
+	}
+
+	// Add finalizer if it does not exist
+	if !controllerutil.ContainsFinalizer(&virtService, serviceFinalizerName) {
+		controllerutil.AddFinalizer(&virtService, serviceFinalizerName)
+		if err := s.virtualClient.Update(ctx, &virtService); err != nil {
+			return reconcile.Result{}, err
+		}
+	}
+	// create or update the service on host
+	if err := s.hostClient.Get(ctx, types.NamespacedName{Name: syncedService.Name, Namespace: s.clusterNamespace}, &hostService); err != nil {
+		if apierrors.IsNotFound(err) {
+			log.Info("creating the service for the first time on the host cluster")
+			return reconcile.Result{}, s.hostClient.Create(ctx, syncedService)
+		}
+		return reconcile.Result{}, err
+	}
+	log.Info("updating service on the host cluster")
+	return reconcile.Result{}, s.hostClient.Update(ctx, syncedService)
+}
+
+func (s *ServiceReconciler) service(obj *v1.Service) *v1.Service {
+	hostService := obj.DeepCopy()
+	s.Translater.TranslateTo(hostService)
+	// don't sync finalizers to the host
+	return hostService
+}

k3k-kubelet/controller/webhook/pod.go

@@ -0,0 +1,120 @@
+package webhook
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/rancher/k3k/pkg/controller/cluster/agent"
+	"github.com/rancher/k3k/pkg/log"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+const (
+	webhookName    = "nodename.podmutator.k3k.io"
+	webhookTimeout = int32(10)
+	webhookPort    = "9443"
+	webhookPath    = "/mutate--v1-pod"
+)
+
+type webhookHandler struct {
+	client           ctrlruntimeclient.Client
+	scheme           *runtime.Scheme
+	nodeName         string
+	clusterName      string
+	clusterNamespace string
+	logger           *log.Logger
+}
+
+// AddPodMutatorWebhook will add a mutator webhook to the virtual cluster to
+// modify the nodeName of the created pods with the name of the virtual kubelet node name
+func AddPodMutatorWebhook(ctx context.Context, mgr manager.Manager, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, nodeName string, logger *log.Logger) error {
+	handler := webhookHandler{
+		client:           mgr.GetClient(),
+		scheme:           mgr.GetScheme(),
+		logger:           logger,
+		clusterName:      clusterName,
+		clusterNamespace: clusterNamespace,
+		nodeName:         nodeName,
+	}
+
+	// create mutator webhook configuration to the cluster
+	config, err := handler.configuration(ctx, hostClient)
+	if err != nil {
+		return err
+	}
+	if err := handler.client.Create(ctx, config); err != nil {
+		return err
+	}
+	// register webhook with the manager
+	return ctrl.NewWebhookManagedBy(mgr).For(&v1.Pod{}).WithDefaulter(&handler).Complete()
+}
+
+func (w *webhookHandler) Default(ctx context.Context, obj runtime.Object) error {
+	pod, ok := obj.(*v1.Pod)
+	if !ok {
+		return fmt.Errorf("invalid request: object was type %t not cluster", obj)
+	}
+	w.logger.Infow("recieved request", "Pod", pod.Name, "Namespace", pod.Namespace)
+	if pod.Spec.NodeName == "" {
+		pod.Spec.NodeName = w.nodeName
+	}
+	return nil
+}
+
+func (w *webhookHandler) configuration(ctx context.Context, hostClient ctrlruntimeclient.Client) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+	w.logger.Infow("extracting webhook tls from host cluster")
+	var (
+		webhookTLSSecret v1.Secret
+	)
+	if err := hostClient.Get(ctx, types.NamespacedName{Name: agent.WebhookSecretName(w.clusterName), Namespace: w.clusterNamespace}, &webhookTLSSecret); err != nil {
+		return nil, err
+	}
+	caBundle, ok := webhookTLSSecret.Data["ca.crt"]
+	if !ok {
+		return nil, errors.New("webhook CABundle does not exist in secret")
+	}
+	webhookURL := "https://" + w.nodeName + ":" + webhookPort + webhookPath
+	return &admissionregistrationv1.MutatingWebhookConfiguration{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "admissionregistration.k8s.io/v1",
+			Kind:       "MutatingWebhookConfiguration",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: webhookName + "-configuration",
+		},
+		Webhooks: []admissionregistrationv1.MutatingWebhook{
+			{
+				Name:                    webhookName,
+				AdmissionReviewVersions: []string{"v1"},
+				SideEffects:             ptr.To(admissionregistrationv1.SideEffectClassNone),
+				TimeoutSeconds:          ptr.To(webhookTimeout),
+				ClientConfig: admissionregistrationv1.WebhookClientConfig{
+					URL:      ptr.To(webhookURL),
+					CABundle: caBundle,
+				},
+				Rules: []admissionregistrationv1.RuleWithOperations{
+					{
+						Operations: []admissionregistrationv1.OperationType{
+							"CREATE",
+						},
+						Rule: admissionregistrationv1.Rule{
+							APIGroups:   []string{""},
+							APIVersions: []string{"v1"},
+							Resources:   []string{"pods"},
+							Scope:       ptr.To(admissionregistrationv1.NamespacedScope),
+						},
+					},
+				},
+			},
+		},
+	}, nil
+}

k3k-kubelet/kubelet.go

@@ -0,0 +1,374 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	certutil "github.com/rancher/dynamiclistener/cert"
+	k3kkubeletcontroller "github.com/rancher/k3k/k3k-kubelet/controller"
+	k3kwebhook "github.com/rancher/k3k/k3k-kubelet/controller/webhook"
+	"github.com/rancher/k3k/k3k-kubelet/provider"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/certs"
+	"github.com/rancher/k3k/pkg/controller/cluster/server"
+	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	k3klog "github.com/rancher/k3k/pkg/log"
+	"github.com/virtual-kubelet/virtual-kubelet/log"
+	"github.com/virtual-kubelet/virtual-kubelet/node"
+	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
+	"go.uber.org/zap"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/client-go/kubernetes"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/client-go/util/retry"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	ctrlserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+)
+
+var (
+	baseScheme     = runtime.NewScheme()
+	k3kKubeletName = "k3k-kubelet"
+)
+
+func init() {
+	_ = clientgoscheme.AddToScheme(baseScheme)
+	_ = v1alpha1.AddToScheme(baseScheme)
+}
+
+type kubelet struct {
+	virtualCluster v1alpha1.Cluster
+
+	name       string
+	port       int
+	hostConfig *rest.Config
+	virtConfig *rest.Config
+	agentIP    string
+	dnsIP      string
+	hostClient ctrlruntimeclient.Client
+	virtClient kubernetes.Interface
+	hostMgr    manager.Manager
+	virtualMgr manager.Manager
+	node       *nodeutil.Node
+	logger     *k3klog.Logger
+	token      string
+}
+
+func newKubelet(ctx context.Context, c *config, logger *k3klog.Logger) (*kubelet, error) {
+	hostConfig, err := clientcmd.BuildConfigFromFlags("", c.HostConfigPath)
+	if err != nil {
+		return nil, err
+	}
+
+	hostClient, err := ctrlruntimeclient.New(hostConfig, ctrlruntimeclient.Options{
+		Scheme: baseScheme,
+	})
+	if err != nil {
+		return nil, err
+	}
+	virtConfig, err := virtRestConfig(ctx, c.VirtualConfigPath, hostClient, c.ClusterName, c.ClusterNamespace, c.Token, logger)
+	if err != nil {
+		return nil, err
+	}
+
+	virtClient, err := kubernetes.NewForConfig(virtConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	hostMgr, err := ctrl.NewManager(hostConfig, manager.Options{
+		Scheme: baseScheme,
+		Metrics: ctrlserver.Options{
+			BindAddress: ":8083",
+		},
+		Cache: cache.Options{
+			DefaultNamespaces: map[string]cache.Config{
+				c.ClusterNamespace: {},
+			},
+		},
+	})
+	if err != nil {
+		return nil, errors.New("unable to create controller-runtime mgr for host cluster: " + err.Error())
+	}
+
+	virtualScheme := runtime.NewScheme()
+	// virtual client will only use core types (for now), no need to add anything other than the basics
+	err = clientgoscheme.AddToScheme(virtualScheme)
+	if err != nil {
+		return nil, errors.New("unable to add client go types to virtual cluster scheme: " + err.Error())
+	}
+	webhookServer := webhook.NewServer(webhook.Options{
+		CertDir: "/opt/rancher/k3k-webhook",
+	})
+	virtualMgr, err := ctrl.NewManager(virtConfig, manager.Options{
+		Scheme:        virtualScheme,
+		WebhookServer: webhookServer,
+		Metrics: ctrlserver.Options{
+			BindAddress: ":8084",
+		},
+	})
+	if err != nil {
+		return nil, errors.New("unable to create controller-runtime mgr for virtual cluster: " + err.Error())
+	}
+	logger.Info("adding pod mutator webhook")
+	if err := k3kwebhook.AddPodMutatorWebhook(ctx, virtualMgr, hostClient, c.ClusterName, c.ClusterNamespace, c.NodeName, logger); err != nil {
+		return nil, errors.New("unable to add pod mutator webhook for virtual cluster: " + err.Error())
+	}
+
+	logger.Info("adding service syncer controller")
+	if err := k3kkubeletcontroller.AddServiceSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace, k3klog.New(false)); err != nil {
+		return nil, errors.New("failed to add service syncer controller: " + err.Error())
+	}
+
+	logger.Info("adding pvc syncer controller")
+	if err := k3kkubeletcontroller.AddPVCSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace, k3klog.New(false)); err != nil {
+		return nil, errors.New("failed to add pvc syncer controller: " + err.Error())
+	}
+
+	clusterIP, err := clusterIP(ctx, c.AgentHostname, c.ClusterNamespace, hostClient)
+	if err != nil {
+		return nil, errors.New("failed to extract the clusterIP for the server service: " + err.Error())
+	}
+
+	// get the cluster's DNS IP to be injected to pods
+	var dnsService v1.Service
+	dnsName := controller.SafeConcatNameWithPrefix(c.ClusterName, "kube-dns")
+	if err := hostClient.Get(ctx, types.NamespacedName{Name: dnsName, Namespace: c.ClusterNamespace}, &dnsService); err != nil {
+		return nil, errors.New("failed to get the DNS service for the cluster: " + err.Error())
+	}
+
+	var virtualCluster v1alpha1.Cluster
+	if err := hostClient.Get(ctx, types.NamespacedName{Name: c.ClusterName, Namespace: c.ClusterNamespace}, &virtualCluster); err != nil {
+		return nil, errors.New("failed to get virtualCluster spec: " + err.Error())
+	}
+
+	return &kubelet{
+		virtualCluster: virtualCluster,
+
+		name:       c.NodeName,
+		hostConfig: hostConfig,
+		hostClient: hostClient,
+		virtConfig: virtConfig,
+		virtClient: virtClient,
+		hostMgr:    hostMgr,
+		virtualMgr: virtualMgr,
+		agentIP:    clusterIP,
+		logger:     logger.Named(k3kKubeletName),
+		token:      c.Token,
+		dnsIP:      dnsService.Spec.ClusterIP,
+	}, nil
+}
+
+func clusterIP(ctx context.Context, serviceName, clusterNamespace string, hostClient ctrlruntimeclient.Client) (string, error) {
+	var service v1.Service
+	serviceKey := types.NamespacedName{Namespace: clusterNamespace, Name: serviceName}
+	if err := hostClient.Get(ctx, serviceKey, &service); err != nil {
+		return "", err
+	}
+	return service.Spec.ClusterIP, nil
+}
+
+func (k *kubelet) registerNode(ctx context.Context, agentIP, srvPort, namespace, name, hostname, serverIP, dnsIP string) error {
+	providerFunc := k.newProviderFunc(namespace, name, hostname, agentIP, serverIP, dnsIP)
+	nodeOpts := k.nodeOpts(ctx, srvPort, namespace, name, hostname, agentIP)
+
+	var err error
+	k.node, err = nodeutil.NewNode(k.name, providerFunc, nodeutil.WithClient(k.virtClient), nodeOpts)
+	if err != nil {
+		return errors.New("unable to start kubelet: " + err.Error())
+	}
+	return nil
+}
+
+func (k *kubelet) start(ctx context.Context) {
+	// any one of the following 3 tasks (host manager, virtual manager, node) crashing will stop the
+	// program, and all 3 of them block on start, so we start them here in go-routines
+	go func() {
+		err := k.hostMgr.Start(ctx)
+		if err != nil {
+			k.logger.Fatalw("host manager stopped", zap.Error(err))
+		}
+	}()
+
+	go func() {
+		err := k.virtualMgr.Start(ctx)
+		if err != nil {
+			k.logger.Fatalw("virtual manager stopped", zap.Error(err))
+		}
+	}()
+
+	// run the node async so that we can wait for it to be ready in another call
+
+	go func() {
+		ctx = log.WithLogger(ctx, k.logger)
+		if err := k.node.Run(ctx); err != nil {
+			k.logger.Fatalw("node errored when running", zap.Error(err))
+		}
+	}()
+
+	if err := k.node.WaitReady(context.Background(), time.Minute*1); err != nil {
+		k.logger.Fatalw("node was not ready within timeout of 1 minute", zap.Error(err))
+	}
+	<-k.node.Done()
+	if err := k.node.Err(); err != nil {
+		k.logger.Fatalw("node stopped with an error", zap.Error(err))
+	}
+	k.logger.Info("node exited successfully")
+}
+
+func (k *kubelet) newProviderFunc(namespace, name, hostname, agentIP, serverIP, dnsIP string) nodeutil.NewProviderFunc {
+	return func(pc nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) {
+		utilProvider, err := provider.New(*k.hostConfig, k.hostMgr, k.virtualMgr, k.logger, namespace, name, serverIP, dnsIP)
+		if err != nil {
+			return nil, nil, errors.New("unable to make nodeutil provider: " + err.Error())
+		}
+
+		provider.ConfigureNode(k.logger, pc.Node, hostname, k.port, agentIP, utilProvider.CoreClient, utilProvider.VirtualClient, k.virtualCluster)
+
+		return utilProvider, &provider.Node{}, nil
+	}
+}
+
+func (k *kubelet) nodeOpts(ctx context.Context, srvPort, namespace, name, hostname, agentIP string) nodeutil.NodeOpt {
+	return func(c *nodeutil.NodeConfig) error {
+		c.HTTPListenAddr = fmt.Sprintf(":%s", srvPort)
+		// set up the routes
+		mux := http.NewServeMux()
+		if err := nodeutil.AttachProviderRoutes(mux)(c); err != nil {
+			return errors.New("unable to attach routes: " + err.Error())
+		}
+		c.Handler = mux
+
+		tlsConfig, err := loadTLSConfig(ctx, k.hostClient, name, namespace, k.name, hostname, k.token, agentIP)
+		if err != nil {
+			return errors.New("unable to get tls config: " + err.Error())
+		}
+		c.TLSConfig = tlsConfig
+		return nil
+	}
+}
+
+func virtRestConfig(ctx context.Context, virtualConfigPath string, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, token string, logger *k3klog.Logger) (*rest.Config, error) {
+	if virtualConfigPath != "" {
+		return clientcmd.BuildConfigFromFlags("", virtualConfigPath)
+	}
+	// virtual kubeconfig file is empty, trying to fetch the k3k cluster kubeconfig
+	var cluster v1alpha1.Cluster
+	if err := hostClient.Get(ctx, types.NamespacedName{Namespace: clusterNamespace, Name: clusterName}, &cluster); err != nil {
+		return nil, err
+	}
+	endpoint := server.ServiceName(cluster.Name) + "." + cluster.Namespace
+	var b *bootstrap.ControlRuntimeBootstrap
+	if err := retry.OnError(controller.Backoff, func(err error) bool {
+		return err != nil
+	}, func() error {
+		var err error
+		b, err = bootstrap.DecodedBootstrap(token, endpoint)
+		logger.Infow("decoded bootstrap", zap.Error(err))
+		return err
+	}); err != nil {
+		return nil, errors.New("unable to decode bootstrap: " + err.Error())
+	}
+	adminCert, adminKey, err := certs.CreateClientCertKey(
+		controller.AdminCommonName, []string{user.SystemPrivilegedGroup},
+		nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, time.Hour*24*time.Duration(356),
+		b.ClientCA.Content,
+		b.ClientCAKey.Content)
+	if err != nil {
+		return nil, err
+	}
+
+	url := fmt.Sprintf("https://%s:%d", server.ServiceName(cluster.Name), server.ServerPort)
+	kubeconfigData, err := kubeconfigBytes(url, []byte(b.ServerCA.Content), adminCert, adminKey)
+	if err != nil {
+		return nil, err
+	}
+	return clientcmd.RESTConfigFromKubeConfig(kubeconfigData)
+}
+
+func kubeconfigBytes(url string, serverCA, clientCert, clientKey []byte) ([]byte, error) {
+	config := clientcmdapi.NewConfig()
+
+	cluster := clientcmdapi.NewCluster()
+	cluster.CertificateAuthorityData = serverCA
+	cluster.Server = url
+
+	authInfo := clientcmdapi.NewAuthInfo()
+	authInfo.ClientCertificateData = clientCert
+	authInfo.ClientKeyData = clientKey
+
+	context := clientcmdapi.NewContext()
+	context.AuthInfo = "default"
+	context.Cluster = "default"
+
+	config.Clusters["default"] = cluster
+	config.AuthInfos["default"] = authInfo
+	config.Contexts["default"] = context
+	config.CurrentContext = "default"
+
+	return clientcmd.Write(*config)
+}
+
+func loadTLSConfig(ctx context.Context, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, nodeName, hostname, token, agentIP string) (*tls.Config, error) {
+	var (
+		cluster v1alpha1.Cluster
+		b       *bootstrap.ControlRuntimeBootstrap
+	)
+	if err := hostClient.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: clusterNamespace}, &cluster); err != nil {
+		return nil, err
+	}
+	endpoint := fmt.Sprintf("%s.%s", server.ServiceName(cluster.Name), cluster.Namespace)
+	if err := retry.OnError(controller.Backoff, func(err error) bool {
+		return err != nil
+	}, func() error {
+		var err error
+		b, err = bootstrap.DecodedBootstrap(token, endpoint)
+		return err
+	}); err != nil {
+		return nil, errors.New("unable to decode bootstrap: " + err.Error())
+	}
+	ip := net.ParseIP(agentIP)
+	altNames := certutil.AltNames{
+		DNSNames: []string{hostname},
+		IPs:      []net.IP{ip},
+	}
+	cert, key, err := certs.CreateClientCertKey(nodeName, nil, &altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, 0, b.ServerCA.Content, b.ServerCAKey.Content)
+	if err != nil {
+		return nil, errors.New("unable to get cert and key: " + err.Error())
+	}
+	clientCert, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, errors.New("unable to get key pair: " + err.Error())
+	}
+	// create rootCA CertPool
+	certs, err := certutil.ParseCertsPEM([]byte(b.ServerCA.Content))
+	if err != nil {
+		return nil, errors.New("unable to create ca certs: " + err.Error())
+	}
+	if len(certs) < 1 {
+		return nil, errors.New("ca cert is not parsed correctly")
+	}
+	pool := x509.NewCertPool()
+	pool.AddCert(certs[0])
+
+	return &tls.Config{
+		RootCAs:      pool,
+		Certificates: []tls.Certificate{clientCert},
+	}, nil
+}

k3k-kubelet/main.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"context"
+	"os"
+
+	"github.com/go-logr/zapr"
+	"github.com/rancher/k3k/pkg/log"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+	"go.uber.org/zap"
+	ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+var (
+	configFile string
+	cfg        config
+	logger     *log.Logger
+	debug      bool
+)
+
+func main() {
+	app := cli.NewApp()
+	app.Name = "k3k-kubelet"
+	app.Usage = "virtual kubelet implementation k3k"
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:        "cluster-name",
+			Usage:       "Name of the k3k cluster",
+			Destination: &cfg.ClusterName,
+			EnvVar:      "CLUSTER_NAME",
+		},
+		cli.StringFlag{
+			Name:        "cluster-namespace",
+			Usage:       "Namespace of the k3k cluster",
+			Destination: &cfg.ClusterNamespace,
+			EnvVar:      "CLUSTER_NAMESPACE",
+		},
+		cli.StringFlag{
+			Name:        "cluster-token",
+			Usage:       "K3S token of the k3k cluster",
+			Destination: &cfg.Token,
+			EnvVar:      "CLUSTER_TOKEN",
+		},
+		cli.StringFlag{
+			Name:        "host-config-path",
+			Usage:       "Path to the host kubeconfig, if empty then virtual-kubelet will use incluster config",
+			Destination: &cfg.HostConfigPath,
+			EnvVar:      "HOST_KUBECONFIG",
+		},
+		cli.StringFlag{
+			Name:        "virtual-config-path",
+			Usage:       "Path to the k3k cluster kubeconfig, if empty then virtual-kubelet will create its own config from k3k cluster",
+			Destination: &cfg.VirtualConfigPath,
+			EnvVar:      "CLUSTER_NAME",
+		},
+		cli.StringFlag{
+			Name:        "kubelet-port",
+			Usage:       "kubelet API port number",
+			Destination: &cfg.KubeletPort,
+			EnvVar:      "SERVER_PORT",
+			Value:       "10250",
+		},
+		cli.StringFlag{
+			Name:        "agent-hostname",
+			Usage:       "Agent Hostname used for TLS SAN for the kubelet server",
+			Destination: &cfg.AgentHostname,
+			EnvVar:      "AGENT_HOSTNAME",
+		},
+		cli.StringFlag{
+			Name:        "server-ip",
+			Usage:       "Server IP used for registering the virtual kubelet to the cluster",
+			Destination: &cfg.ServerIP,
+			EnvVar:      "SERVER_IP",
+		},
+		cli.StringFlag{
+			Name:        "config",
+			Usage:       "Path to k3k-kubelet config file",
+			Destination: &configFile,
+			EnvVar:      "CONFIG_FILE",
+			Value:       "/etc/rancher/k3k/config.yaml",
+		},
+		cli.BoolFlag{
+			Name:        "debug",
+			Usage:       "Enable debug logging",
+			Destination: &debug,
+			EnvVar:      "DEBUG",
+		},
+	}
+	app.Before = func(clx *cli.Context) error {
+		logger = log.New(debug)
+		ctrlruntimelog.SetLogger(zapr.NewLogger(logger.Desugar().WithOptions(zap.AddCallerSkip(1))))
+		return nil
+	}
+	app.Action = run
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+func run(clx *cli.Context) {
+	ctx := context.Background()
+	if err := cfg.parse(configFile); err != nil {
+		logger.Fatalw("failed to parse config file", "path", configFile, zap.Error(err))
+	}
+
+	if err := cfg.validate(); err != nil {
+		logger.Fatalw("failed to validate config", zap.Error(err))
+	}
+	k, err := newKubelet(ctx, &cfg, logger)
+	if err != nil {
+		logger.Fatalw("failed to create new virtual kubelet instance", zap.Error(err))
+	}
+
+	if err := k.registerNode(ctx, k.agentIP, cfg.KubeletPort, cfg.ClusterNamespace, cfg.ClusterName, cfg.AgentHostname, cfg.ServerIP, k.dnsIP); err != nil {
+		logger.Fatalw("failed to register new node", zap.Error(err))
+	}
+
+	k.start(ctx)
+}

k3k-kubelet/provider/collectors/kubelet_resource_metrics.go

@@ -0,0 +1,196 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the Apache 2.0 license.
+
+See https://github.com/virtual-kubelet/azure-aci/tree/master/pkg/metrics/collectors
+*/
+
+package collectors
+
+import (
+	"time"
+
+	stats "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
+	compbasemetrics "k8s.io/component-base/metrics"
+)
+
+// defining metrics
+var (
+	nodeCPUUsageDesc = compbasemetrics.NewDesc("node_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the node in core-seconds",
+		nil,
+		nil,
+		compbasemetrics.ALPHA,
+		"")
+
+	nodeMemoryUsageDesc = compbasemetrics.NewDesc("node_memory_working_set_bytes",
+		"Current working set of the node in bytes",
+		nil,
+		nil,
+		compbasemetrics.ALPHA,
+		"")
+
+	containerCPUUsageDesc = compbasemetrics.NewDesc("container_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the container in core-seconds",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		compbasemetrics.ALPHA,
+		"")
+
+	containerMemoryUsageDesc = compbasemetrics.NewDesc("container_memory_working_set_bytes",
+		"Current working set of the container in bytes",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		compbasemetrics.ALPHA,
+		"")
+
+	podCPUUsageDesc = compbasemetrics.NewDesc("pod_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the pod in core-seconds",
+		[]string{"pod", "namespace"},
+		nil,
+		compbasemetrics.ALPHA,
+		"")
+
+	podMemoryUsageDesc = compbasemetrics.NewDesc("pod_memory_working_set_bytes",
+		"Current working set of the pod in bytes",
+		[]string{"pod", "namespace"},
+		nil,
+		compbasemetrics.ALPHA,
+		"")
+
+	resourceScrapeResultDesc = compbasemetrics.NewDesc("scrape_error",
+		"1 if there was an error while getting container metrics, 0 otherwise",
+		nil,
+		nil,
+		compbasemetrics.ALPHA,
+		"")
+
+	containerStartTimeDesc = compbasemetrics.NewDesc("container_start_time_seconds",
+		"Start time of the container since unix epoch in seconds",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		compbasemetrics.ALPHA,
+		"")
+)
+
+// NewResourceMetricsCollector returns a metrics.StableCollector which exports resource metrics
+func NewKubeletResourceMetricsCollector(podStats *stats.Summary) compbasemetrics.StableCollector {
+	return &resourceMetricsCollector{
+		providerPodStats: podStats,
+	}
+}
+
+type resourceMetricsCollector struct {
+	compbasemetrics.BaseStableCollector
+
+	providerPodStats *stats.Summary
+}
+
+// Check if resourceMetricsCollector implements necessary interface
+var _ compbasemetrics.StableCollector = &resourceMetricsCollector{}
+
+// DescribeWithStability implements compbasemetrics.StableCollector
+func (rc *resourceMetricsCollector) DescribeWithStability(ch chan<- *compbasemetrics.Desc) {
+	ch <- nodeCPUUsageDesc
+	ch <- nodeMemoryUsageDesc
+	ch <- containerStartTimeDesc
+	ch <- containerCPUUsageDesc
+	ch <- containerMemoryUsageDesc
+	ch <- podCPUUsageDesc
+	ch <- podMemoryUsageDesc
+	ch <- resourceScrapeResultDesc
+}
+
+// CollectWithStability implements compbasemetrics.StableCollector
+// Since new containers are frequently created and removed, using the Gauge would
+// leak metric collectors for containers or pods that no longer exist.  Instead, implement
+// custom collector in a way that only collects metrics for active containers.
+func (rc *resourceMetricsCollector) CollectWithStability(ch chan<- compbasemetrics.Metric) {
+	var errorCount float64
+	defer func() {
+		ch <- compbasemetrics.NewLazyConstMetric(resourceScrapeResultDesc, compbasemetrics.GaugeValue, errorCount)
+	}()
+
+	statsSummary := *rc.providerPodStats
+	rc.collectNodeCPUMetrics(ch, statsSummary.Node)
+	rc.collectNodeMemoryMetrics(ch, statsSummary.Node)
+
+	for _, pod := range statsSummary.Pods {
+		for _, container := range pod.Containers {
+			rc.collectContainerStartTime(ch, pod, container)
+			rc.collectContainerCPUMetrics(ch, pod, container)
+			rc.collectContainerMemoryMetrics(ch, pod, container)
+		}
+		rc.collectPodCPUMetrics(ch, pod)
+		rc.collectPodMemoryMetrics(ch, pod)
+	}
+}
+
+// implement collector methods and validate that correct data is used
+
+func (rc *resourceMetricsCollector) collectNodeCPUMetrics(ch chan<- compbasemetrics.Metric, s stats.NodeStats) {
+	if s.CPU == nil || s.CPU.UsageCoreNanoSeconds == nil {
+		return
+	}
+
+	ch <- compbasemetrics.NewLazyMetricWithTimestamp(s.CPU.Time.Time,
+		compbasemetrics.NewLazyConstMetric(nodeCPUUsageDesc, compbasemetrics.CounterValue, float64(*s.CPU.UsageCoreNanoSeconds)/float64(time.Second)))
+}
+
+func (rc *resourceMetricsCollector) collectNodeMemoryMetrics(ch chan<- compbasemetrics.Metric, s stats.NodeStats) {
+	if s.Memory == nil || s.Memory.WorkingSetBytes == nil {
+		return
+	}
+
+	ch <- compbasemetrics.NewLazyMetricWithTimestamp(s.Memory.Time.Time,
+		compbasemetrics.NewLazyConstMetric(nodeMemoryUsageDesc, compbasemetrics.GaugeValue, float64(*s.Memory.WorkingSetBytes)))
+}
+
+func (rc *resourceMetricsCollector) collectContainerStartTime(ch chan<- compbasemetrics.Metric, pod stats.PodStats, s stats.ContainerStats) {
+	if s.StartTime.Unix() <= 0 {
+		return
+	}
+
+	ch <- compbasemetrics.NewLazyMetricWithTimestamp(s.StartTime.Time,
+		compbasemetrics.NewLazyConstMetric(containerStartTimeDesc, compbasemetrics.GaugeValue, float64(s.StartTime.UnixNano())/float64(time.Second), s.Name, pod.PodRef.Name, pod.PodRef.Namespace))
+}
+
+func (rc *resourceMetricsCollector) collectContainerCPUMetrics(ch chan<- compbasemetrics.Metric, pod stats.PodStats, s stats.ContainerStats) {
+	if s.CPU == nil || s.CPU.UsageCoreNanoSeconds == nil {
+		return
+	}
+
+	ch <- compbasemetrics.NewLazyMetricWithTimestamp(s.CPU.Time.Time,
+		compbasemetrics.NewLazyConstMetric(containerCPUUsageDesc, compbasemetrics.CounterValue,
+			float64(*s.CPU.UsageCoreNanoSeconds)/float64(time.Second), s.Name, pod.PodRef.Name, pod.PodRef.Namespace))
+}
+
+func (rc *resourceMetricsCollector) collectContainerMemoryMetrics(ch chan<- compbasemetrics.Metric, pod stats.PodStats, s stats.ContainerStats) {
+	if s.Memory == nil || s.Memory.WorkingSetBytes == nil {
+		return
+	}
+
+	ch <- compbasemetrics.NewLazyMetricWithTimestamp(s.Memory.Time.Time,
+		compbasemetrics.NewLazyConstMetric(containerMemoryUsageDesc, compbasemetrics.GaugeValue,
+			float64(*s.Memory.WorkingSetBytes), s.Name, pod.PodRef.Name, pod.PodRef.Namespace))
+}
+
+func (rc *resourceMetricsCollector) collectPodCPUMetrics(ch chan<- compbasemetrics.Metric, pod stats.PodStats) {
+	if pod.CPU == nil || pod.CPU.UsageCoreNanoSeconds == nil {
+		return
+	}
+
+	ch <- compbasemetrics.NewLazyMetricWithTimestamp(pod.CPU.Time.Time,
+		compbasemetrics.NewLazyConstMetric(podCPUUsageDesc, compbasemetrics.CounterValue,
+			float64(*pod.CPU.UsageCoreNanoSeconds)/float64(time.Second), pod.PodRef.Name, pod.PodRef.Namespace))
+}
+
+func (rc *resourceMetricsCollector) collectPodMemoryMetrics(ch chan<- compbasemetrics.Metric, pod stats.PodStats) {
+	if pod.Memory == nil || pod.Memory.WorkingSetBytes == nil {
+		return
+	}
+
+	ch <- compbasemetrics.NewLazyMetricWithTimestamp(pod.Memory.Time.Time,
+		compbasemetrics.NewLazyConstMetric(podMemoryUsageDesc, compbasemetrics.GaugeValue,
+			float64(*pod.Memory.WorkingSetBytes), pod.PodRef.Name, pod.PodRef.Namespace))
+}

k3k-kubelet/provider/configure.go

@@ -0,0 +1,163 @@
+package provider
+
+import (
+	"context"
+	"time"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	k3klog "github.com/rancher/k3k/pkg/log"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
+	typedv1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func ConfigureNode(logger *k3klog.Logger, node *v1.Node, hostname string, servicePort int, ip string, coreClient typedv1.CoreV1Interface, virtualClient client.Client, virtualCluster v1alpha1.Cluster) {
+	node.Status.Conditions = nodeConditions()
+	node.Status.DaemonEndpoints.KubeletEndpoint.Port = int32(servicePort)
+	node.Status.Addresses = []v1.NodeAddress{
+		{
+			Type:    v1.NodeHostName,
+			Address: hostname,
+		},
+		{
+			Type:    v1.NodeInternalIP,
+			Address: ip,
+		},
+	}
+
+	node.Labels["node.kubernetes.io/exclude-from-external-load-balancers"] = "true"
+	node.Labels["kubernetes.io/os"] = "linux"
+
+	updateNodeCapacityInterval := 10 * time.Second
+	ticker := time.NewTicker(updateNodeCapacityInterval)
+
+	go func() {
+		for range ticker.C {
+			if err := updateNodeCapacity(coreClient, virtualClient, node.Name, virtualCluster.Spec.NodeSelector); err != nil {
+				logger.Error("error updating node capacity", err)
+			}
+		}
+	}()
+}
+
+// nodeConditions returns the basic conditions which mark the node as ready
+func nodeConditions() []v1.NodeCondition {
+	return []v1.NodeCondition{
+		{
+			Type:               "Ready",
+			Status:             v1.ConditionTrue,
+			LastHeartbeatTime:  metav1.Now(),
+			LastTransitionTime: metav1.Now(),
+			Reason:             "KubeletReady",
+			Message:            "kubelet is ready.",
+		},
+		{
+			Type:               "OutOfDisk",
+			Status:             v1.ConditionFalse,
+			LastHeartbeatTime:  metav1.Now(),
+			LastTransitionTime: metav1.Now(),
+			Reason:             "KubeletHasSufficientDisk",
+			Message:            "kubelet has sufficient disk space available",
+		},
+		{
+			Type:               "MemoryPressure",
+			Status:             v1.ConditionFalse,
+			LastHeartbeatTime:  metav1.Now(),
+			LastTransitionTime: metav1.Now(),
+			Reason:             "KubeletHasSufficientMemory",
+			Message:            "kubelet has sufficient memory available",
+		},
+		{
+			Type:               "DiskPressure",
+			Status:             v1.ConditionFalse,
+			LastHeartbeatTime:  metav1.Now(),
+			LastTransitionTime: metav1.Now(),
+			Reason:             "KubeletHasNoDiskPressure",
+			Message:            "kubelet has no disk pressure",
+		},
+		{
+			Type:               "NetworkUnavailable",
+			Status:             v1.ConditionFalse,
+			LastHeartbeatTime:  metav1.Now(),
+			LastTransitionTime: metav1.Now(),
+			Reason:             "RouteCreated",
+			Message:            "RouteController created a route",
+		},
+	}
+}
+
+// updateNodeCapacity will update the virtual node capacity (and the allocatable field) with the sum of all the resource in the host nodes.
+// If the nodeLabels are specified only the matching nodes will be considered.
+func updateNodeCapacity(coreClient typedv1.CoreV1Interface, virtualClient client.Client, virtualNodeName string, nodeLabels map[string]string) error {
+	ctx := context.Background()
+
+	capacity, allocatable, err := getResourcesFromNodes(ctx, coreClient, nodeLabels)
+	if err != nil {
+		return err
+	}
+
+	var virtualNode corev1.Node
+	if err := virtualClient.Get(ctx, types.NamespacedName{Name: virtualNodeName}, &virtualNode); err != nil {
+		return err
+	}
+
+	virtualNode.Status.Capacity = capacity
+	virtualNode.Status.Allocatable = allocatable
+
+	return virtualClient.Status().Update(ctx, &virtualNode)
+}
+
+// getResourcesFromNodes will return a sum of all the resource capacity of the host nodes, and the allocatable resources.
+// If some node labels are specified only the matching nodes will be considered.
+func getResourcesFromNodes(ctx context.Context, coreClient typedv1.CoreV1Interface, nodeLabels map[string]string) (v1.ResourceList, v1.ResourceList, error) {
+	listOpts := metav1.ListOptions{}
+	if nodeLabels != nil {
+		labelSelector := metav1.LabelSelector{MatchLabels: nodeLabels}
+		listOpts.LabelSelector = labels.Set(labelSelector.MatchLabels).String()
+	}
+
+	nodeList, err := coreClient.Nodes().List(ctx, listOpts)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// sum all
+	virtualCapacityResources := corev1.ResourceList{}
+	virtualAvailableResources := corev1.ResourceList{}
+
+	for _, node := range nodeList.Items {
+
+		// check if the node is Ready
+		for _, condition := range node.Status.Conditions {
+			if condition.Type != corev1.NodeReady {
+				continue
+			}
+
+			// if the node is not Ready then we can skip it
+			if condition.Status != corev1.ConditionTrue {
+				break
+			}
+		}
+
+		// add all the available metrics to the virtual node
+		for resourceName, resourceQuantity := range node.Status.Capacity {
+			virtualResource := virtualCapacityResources[resourceName]
+
+			(&virtualResource).Add(resourceQuantity)
+			virtualCapacityResources[resourceName] = virtualResource
+		}
+
+		for resourceName, resourceQuantity := range node.Status.Allocatable {
+			virtualResource := virtualAvailableResources[resourceName]
+
+			(&virtualResource).Add(resourceQuantity)
+			virtualAvailableResources[resourceName] = virtualResource
+		}
+	}
+
+	return virtualCapacityResources, virtualAvailableResources, nil
+}

k3k-kubelet/provider/node.go

@@ -0,0 +1,22 @@
+package provider
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// Node implements the node.Provider interface from Virtual Kubelet
+type Node struct {
+	notifyCallback func(*corev1.Node)
+}
+
+// Ping is called to check if the node is healthy - in the current format it always is
+func (n *Node) Ping(context.Context) error {
+	return nil
+}
+
+// NotifyNodeStatus sets the callback function for a node being changed. As of now, no changes are made
+func (n *Node) NotifyNodeStatus(ctx context.Context, cb func(*corev1.Node)) {
+	n.notifyCallback = cb
+}

k3k-kubelet/provider/provider.go

@@ -0,0 +1,667 @@
+package provider
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/rancher/k3k/k3k-kubelet/controller"
+	"github.com/rancher/k3k/k3k-kubelet/provider/collectors"
+	"github.com/rancher/k3k/k3k-kubelet/translate"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	k3klog "github.com/rancher/k3k/pkg/log"
+	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	"github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/kubernetes/scheme"
+	cv1 "k8s.io/client-go/kubernetes/typed/core/v1"
+
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/portforward"
+	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/client-go/transport/spdy"
+	compbasemetrics "k8s.io/component-base/metrics"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+// Provider implements nodetuil.Provider from virtual Kubelet.
+// TODO: Implement NotifyPods and the required usage so that this can be an async provider
+type Provider struct {
+	Handler          controller.ControllerHandler
+	Translater       translate.ToHostTranslater
+	HostClient       client.Client
+	VirtualClient    client.Client
+	ClientConfig     rest.Config
+	CoreClient       cv1.CoreV1Interface
+	ClusterNamespace string
+	ClusterName      string
+	serverIP         string
+	dnsIP            string
+	logger           *k3klog.Logger
+}
+
+func New(hostConfig rest.Config, hostMgr, virtualMgr manager.Manager, logger *k3klog.Logger, namespace, name, serverIP, dnsIP string) (*Provider, error) {
+	coreClient, err := cv1.NewForConfig(&hostConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	translater := translate.ToHostTranslater{
+		ClusterName:      name,
+		ClusterNamespace: namespace,
+	}
+
+	p := Provider{
+		Handler: controller.ControllerHandler{
+			Mgr:           virtualMgr,
+			Scheme:        *virtualMgr.GetScheme(),
+			HostClient:    hostMgr.GetClient(),
+			VirtualClient: virtualMgr.GetClient(),
+			Translater:    translater,
+			Logger:        logger,
+		},
+		HostClient:       hostMgr.GetClient(),
+		VirtualClient:    virtualMgr.GetClient(),
+		Translater:       translater,
+		ClientConfig:     hostConfig,
+		CoreClient:       coreClient,
+		ClusterNamespace: namespace,
+		ClusterName:      name,
+		logger:           logger,
+		serverIP:         serverIP,
+		dnsIP:            dnsIP,
+	}
+
+	return &p, nil
+}
+
+// GetContainerLogs retrieves the logs of a container by name from the provider.
+func (p *Provider) GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error) {
+	hostPodName := p.Translater.TranslateName(namespace, podName)
+	options := corev1.PodLogOptions{
+		Container:  containerName,
+		Timestamps: opts.Timestamps,
+		Follow:     opts.Follow,
+		Previous:   opts.Previous,
+	}
+	if opts.Tail != 0 {
+		tailLines := int64(opts.Tail)
+		options.TailLines = &tailLines
+	}
+	if opts.LimitBytes != 0 {
+		limitBytes := int64(opts.LimitBytes)
+		options.LimitBytes = &limitBytes
+	}
+	if opts.SinceSeconds != 0 {
+		sinceSeconds := int64(opts.SinceSeconds)
+		options.SinceSeconds = &sinceSeconds
+	}
+	if !opts.SinceTime.IsZero() {
+		sinceTime := metav1.NewTime(opts.SinceTime)
+		options.SinceTime = &sinceTime
+	}
+	closer, err := p.CoreClient.Pods(p.ClusterNamespace).GetLogs(hostPodName, &options).Stream(ctx)
+	p.logger.Infof("got error %s when getting logs for %s in %s", err, hostPodName, p.ClusterNamespace)
+	return closer, err
+}
+
+// RunInContainer executes a command in a container in the pod, copying data
+// between in/out/err and the container's stdin/stdout/stderr.
+func (p *Provider) RunInContainer(ctx context.Context, namespace, podName, containerName string, cmd []string, attach api.AttachIO) error {
+	hostPodName := p.Translater.TranslateName(namespace, podName)
+	req := p.CoreClient.RESTClient().Post().
+		Resource("pods").
+		Name(hostPodName).
+		Namespace(p.ClusterNamespace).
+		SubResource("exec")
+	req.VersionedParams(&corev1.PodExecOptions{
+		Container: containerName,
+		Command:   cmd,
+		TTY:       attach.TTY(),
+		Stdin:     attach.Stdin() != nil,
+		Stdout:    attach.Stdout() != nil,
+		Stderr:    attach.Stderr() != nil,
+	}, scheme.ParameterCodec)
+	exec, err := remotecommand.NewSPDYExecutor(&p.ClientConfig, http.MethodPost, req.URL())
+	if err != nil {
+		return err
+	}
+	return exec.StreamWithContext(ctx, remotecommand.StreamOptions{
+		Stdin:  attach.Stdin(),
+		Stdout: attach.Stdout(),
+		Stderr: attach.Stderr(),
+		Tty:    attach.TTY(),
+		TerminalSizeQueue: &translatorSizeQueue{
+			resizeChan: attach.Resize(),
+		},
+	})
+}
+
+// AttachToContainer attaches to the executing process of a container in the pod, copying data
+// between in/out/err and the container's stdin/stdout/stderr.
+func (p *Provider) AttachToContainer(ctx context.Context, namespace, podName, containerName string, attach api.AttachIO) error {
+	hostPodName := p.Translater.TranslateName(namespace, podName)
+	req := p.CoreClient.RESTClient().Post().
+		Resource("pods").
+		Name(hostPodName).
+		Namespace(p.ClusterNamespace).
+		SubResource("attach")
+	req.VersionedParams(&corev1.PodAttachOptions{
+		Container: containerName,
+		TTY:       attach.TTY(),
+		Stdin:     attach.Stdin() != nil,
+		Stdout:    attach.Stdout() != nil,
+		Stderr:    attach.Stderr() != nil,
+	}, scheme.ParameterCodec)
+	exec, err := remotecommand.NewSPDYExecutor(&p.ClientConfig, http.MethodPost, req.URL())
+	if err != nil {
+		return err
+	}
+	return exec.StreamWithContext(ctx, remotecommand.StreamOptions{
+		Stdin:  attach.Stdin(),
+		Stdout: attach.Stdout(),
+		Stderr: attach.Stderr(),
+		Tty:    attach.TTY(),
+		TerminalSizeQueue: &translatorSizeQueue{
+			resizeChan: attach.Resize(),
+		},
+	})
+}
+
+// GetStatsSummary gets the stats for the node, including running pods
+func (p *Provider) GetStatsSummary(ctx context.Context) (*statsv1alpha1.Summary, error) {
+	p.logger.Debug("GetStatsSummary")
+
+	nodeList := &v1.NodeList{}
+	if err := p.CoreClient.RESTClient().Get().Resource("nodes").Do(ctx).Into(nodeList); err != nil {
+		return nil, fmt.Errorf("unable to get nodes of cluster %s in namespace %s: %w", p.ClusterName, p.ClusterNamespace, err)
+	}
+
+	// fetch the stats from all the nodes
+	var nodeStats statsv1alpha1.NodeStats
+	var allPodsStats []statsv1alpha1.PodStats
+
+	for _, n := range nodeList.Items {
+		res, err := p.CoreClient.RESTClient().
+			Get().
+			Resource("nodes").
+			Name(n.Name).
+			SubResource("proxy").
+			Suffix("stats/summary").
+			DoRaw(ctx)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"unable to get stats of node '%s', from cluster %s in namespace %s: %w",
+				n.Name, p.ClusterName, p.ClusterNamespace, err,
+			)
+		}
+
+		stats := &statsv1alpha1.Summary{}
+		if err := json.Unmarshal(res, stats); err != nil {
+			return nil, err
+		}
+
+		// TODO: we should probably calculate somehow the node stats from the different nodes of the host
+		// or reflect different nodes from the virtual kubelet.
+		// For the moment let's just pick one random node stats.
+		nodeStats = stats.Node
+		allPodsStats = append(allPodsStats, stats.Pods...)
+	}
+
+	pods, err := p.GetPods(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	podsNameMap := make(map[string]*v1.Pod)
+	for _, pod := range pods {
+		hostPodName := p.Translater.TranslateName(pod.Namespace, pod.Name)
+		podsNameMap[hostPodName] = pod
+	}
+
+	filteredStats := &statsv1alpha1.Summary{
+		Node: nodeStats,
+		Pods: make([]statsv1alpha1.PodStats, 0),
+	}
+
+	for _, podStat := range allPodsStats {
+		// skip pods that are not in the cluster namespace
+		if podStat.PodRef.Namespace != p.ClusterNamespace {
+			continue
+		}
+
+		// rewrite the PodReference to match the data of the virtual cluster
+		if pod, found := podsNameMap[podStat.PodRef.Name]; found {
+			podStat.PodRef = statsv1alpha1.PodReference{
+				Name:      pod.Name,
+				Namespace: pod.Namespace,
+				UID:       string(pod.UID),
+			}
+			filteredStats.Pods = append(filteredStats.Pods, podStat)
+		}
+	}
+
+	return filteredStats, nil
+}
+
+// GetMetricsResource gets the metrics for the node, including running pods
+func (p *Provider) GetMetricsResource(ctx context.Context) ([]*dto.MetricFamily, error) {
+	statsSummary, err := p.GetStatsSummary(ctx)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error fetching MetricsResource")
+	}
+
+	registry := compbasemetrics.NewKubeRegistry()
+	registry.CustomMustRegister(collectors.NewKubeletResourceMetricsCollector(statsSummary))
+
+	metricFamily, err := registry.Gather()
+	if err != nil {
+		return nil, errors.Wrapf(err, "error gathering metrics from collector")
+	}
+	return metricFamily, nil
+}
+
+// PortForward forwards a local port to a port on the pod
+func (p *Provider) PortForward(ctx context.Context, namespace, pod string, port int32, stream io.ReadWriteCloser) error {
+	hostPodName := p.Translater.TranslateName(namespace, pod)
+	req := p.CoreClient.RESTClient().Post().
+		Resource("pods").
+		Name(hostPodName).
+		Namespace(p.ClusterNamespace).
+		SubResource("portforward")
+
+	transport, upgrader, err := spdy.RoundTripperFor(&p.ClientConfig)
+	if err != nil {
+		return err
+	}
+	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, req.URL())
+	portAsString := strconv.Itoa(int(port))
+
+	readyChannel := make(chan struct{})
+	stopChannel := make(chan struct{}, 1)
+
+	// Today this doesn't work properly. When the port ward is supposed to stop, the caller (this provider)
+	// should send a value on stopChannel so that the PortForward is stopped. However, we only have a ReadWriteCloser
+	// so more work is needed to detect a close and handle that appropriately.
+	fw, err := portforward.New(dialer, []string{portAsString}, stopChannel, readyChannel, stream, stream)
+
+	if err != nil {
+		return err
+	}
+
+	return fw.ForwardPorts()
+}
+
+// CreatePod takes a Kubernetes Pod and deploys it within the provider.
+func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
+	tPod := pod.DeepCopy()
+	p.Translater.TranslateTo(tPod)
+
+	// get Cluster definition
+	clusterKey := types.NamespacedName{
+		Namespace: p.ClusterNamespace,
+		Name:      p.ClusterName,
+	}
+	var cluster v1alpha1.Cluster
+	if err := p.HostClient.Get(ctx, clusterKey, &cluster); err != nil {
+		return fmt.Errorf("unable to get cluster %s in namespace %s: %w", p.ClusterName, p.ClusterNamespace, err)
+	}
+
+	// these values shouldn't be set on create
+	tPod.UID = ""
+	tPod.ResourceVersion = ""
+
+	// the node was scheduled on the virtual kubelet, but leaving it this way will make it pending indefinitely
+	tPod.Spec.NodeName = ""
+
+	tPod.Spec.NodeSelector = cluster.Spec.NodeSelector
+
+	// if the priorityCluss for the virtual cluster is set then override the provided value
+	// Note: the core-dns and local-path-provisioner pod are scheduled by k3s with the
+	// 'system-cluster-critical' and 'system-node-critical' default priority classes.
+	if cluster.Spec.PriorityClass != "" {
+		tPod.Spec.PriorityClassName = cluster.Spec.PriorityClass
+		tPod.Spec.Priority = nil
+	}
+
+	// volumes will often refer to resources in the virtual cluster, but instead need to refer to the sync'd
+	// host cluster version
+	if err := p.transformVolumes(ctx, pod.Namespace, tPod.Spec.Volumes); err != nil {
+		return fmt.Errorf("unable to sync volumes for pod %s/%s: %w", pod.Namespace, pod.Name, err)
+	}
+	// sync serviceaccount token to a the host cluster
+	if err := p.transformTokens(ctx, pod, tPod); err != nil {
+		return fmt.Errorf("unable to transform tokens for pod %s/%s: %w", pod.Namespace, pod.Name, err)
+	}
+	// inject networking information to the pod including the virtual cluster controlplane endpoint
+	p.configureNetworking(pod.Name, pod.Namespace, tPod)
+
+	p.logger.Infow("Creating pod", "Host Namespace", tPod.Namespace, "Host Name", tPod.Name,
+		"Virtual Namespace", pod.Namespace, "Virtual Name", "env", pod.Name, pod.Spec.Containers[0].Env)
+	return p.HostClient.Create(ctx, tPod)
+}
+
+// transformVolumes changes the volumes to the representation in the host cluster. Will return an error
+// if one/more volumes couldn't be transformed
+func (p *Provider) transformVolumes(ctx context.Context, podNamespace string, volumes []corev1.Volume) error {
+	for _, volume := range volumes {
+		var optional bool
+		if strings.HasPrefix(volume.Name, kubeAPIAccessPrefix) {
+			continue
+		}
+		// note: this needs to handle downward api volumes as well, but more thought is needed on how to do that
+		if volume.ConfigMap != nil {
+			if volume.ConfigMap.Optional != nil {
+				optional = *volume.ConfigMap.Optional
+			}
+			if err := p.syncConfigmap(ctx, podNamespace, volume.ConfigMap.Name, optional); err != nil {
+				return fmt.Errorf("unable to sync configmap volume %s: %w", volume.Name, err)
+			}
+			volume.ConfigMap.Name = p.Translater.TranslateName(podNamespace, volume.ConfigMap.Name)
+		} else if volume.Secret != nil {
+			if volume.Secret.Optional != nil {
+				optional = *volume.Secret.Optional
+			}
+			if err := p.syncSecret(ctx, podNamespace, volume.Secret.SecretName, optional); err != nil {
+				return fmt.Errorf("unable to sync secret volume %s: %w", volume.Name, err)
+			}
+			volume.Secret.SecretName = p.Translater.TranslateName(podNamespace, volume.Secret.SecretName)
+		} else if volume.Projected != nil {
+			for _, source := range volume.Projected.Sources {
+				if source.ConfigMap != nil {
+					if source.ConfigMap.Optional != nil {
+						optional = *source.ConfigMap.Optional
+					}
+					configMapName := source.ConfigMap.Name
+					if err := p.syncConfigmap(ctx, podNamespace, configMapName, optional); err != nil {
+						return fmt.Errorf("unable to sync projected configmap %s: %w", configMapName, err)
+					}
+					source.ConfigMap.Name = p.Translater.TranslateName(podNamespace, configMapName)
+				} else if source.Secret != nil {
+					if source.Secret.Optional != nil {
+						optional = *source.Secret.Optional
+					}
+					secretName := source.Secret.Name
+					if err := p.syncSecret(ctx, podNamespace, secretName, optional); err != nil {
+						return fmt.Errorf("unable to sync projected secret %s: %w", secretName, err)
+					}
+				}
+			}
+		} else if volume.PersistentVolumeClaim != nil {
+			volume.PersistentVolumeClaim.ClaimName = p.Translater.TranslateName(podNamespace, volume.PersistentVolumeClaim.ClaimName)
+		}
+	}
+	return nil
+}
+
+func (p *Provider) syncConfigmap(ctx context.Context, podNamespace string, configMapName string, optional bool) error {
+	var configMap corev1.ConfigMap
+	nsName := types.NamespacedName{
+		Namespace: podNamespace,
+		Name:      configMapName,
+	}
+	err := p.VirtualClient.Get(ctx, nsName, &configMap)
+	if err != nil {
+		// check if its optional configmap
+		if apierrors.IsNotFound(err) && optional {
+			return nil
+		}
+		return fmt.Errorf("unable to get configmap to sync %s/%s: %w", nsName.Namespace, nsName.Name, err)
+	}
+	err = p.Handler.AddResource(ctx, &configMap)
+	if err != nil {
+		return fmt.Errorf("unable to add configmap to sync %s/%s: %w", nsName.Namespace, nsName.Name, err)
+	}
+	return nil
+}
+
+func (p *Provider) syncSecret(ctx context.Context, podNamespace string, secretName string, optional bool) error {
+	var secret corev1.Secret
+	nsName := types.NamespacedName{
+		Namespace: podNamespace,
+		Name:      secretName,
+	}
+	err := p.VirtualClient.Get(ctx, nsName, &secret)
+	if err != nil {
+		if apierrors.IsNotFound(err) && optional {
+			return nil
+		}
+		return fmt.Errorf("unable to get secret to sync %s/%s: %w", nsName.Namespace, nsName.Name, err)
+	}
+	err = p.Handler.AddResource(ctx, &secret)
+	if err != nil {
+		return fmt.Errorf("unable to add configmap to sync %s/%s: %w", nsName.Namespace, nsName.Name, err)
+	}
+	return nil
+}
+
+// UpdatePod takes a Kubernetes Pod and updates it within the provider.
+func (p *Provider) UpdatePod(ctx context.Context, pod *corev1.Pod) error {
+	hostName := p.Translater.TranslateName(pod.Namespace, pod.Name)
+	currentPod, err := p.GetPod(ctx, p.ClusterNamespace, hostName)
+	if err != nil {
+		return fmt.Errorf("unable to get current pod for update: %w", err)
+	}
+	tPod := pod.DeepCopy()
+	p.Translater.TranslateTo(tPod)
+	tPod.UID = currentPod.UID
+	// this is a bit dangerous since another process could have made changes that the user didn't know about
+	tPod.ResourceVersion = currentPod.ResourceVersion
+
+	// Volumes may refer to resources (configmaps/secrets) from the host cluster
+	// So we need the configuration as calculated during create time
+	tPod.Spec.Volumes = currentPod.Spec.Volumes
+	tPod.Spec.Containers = currentPod.Spec.Containers
+	tPod.Spec.InitContainers = currentPod.Spec.InitContainers
+	tPod.Spec.NodeName = currentPod.Spec.NodeName
+
+	return p.HostClient.Update(ctx, tPod)
+}
+
+// DeletePod takes a Kubernetes Pod and deletes it from the provider. Once a pod is deleted, the provider is
+// expected to call the NotifyPods callback with a terminal pod status where all the containers are in a terminal
+// state, as well as the pod. DeletePod may be called multiple times for the same pod.
+func (p *Provider) DeletePod(ctx context.Context, pod *corev1.Pod) error {
+	p.logger.Infof("Got request to delete pod %s", pod.Name)
+	hostName := p.Translater.TranslateName(pod.Namespace, pod.Name)
+	err := p.CoreClient.Pods(p.ClusterNamespace).Delete(ctx, hostName, metav1.DeleteOptions{})
+	if err != nil {
+		return fmt.Errorf("unable to delete pod %s/%s: %w", pod.Namespace, pod.Name, err)
+	}
+	if err = p.pruneUnusedVolumes(ctx, pod); err != nil {
+		// note that we don't return an error here. The pod was sucessfully deleted, another process
+		// should clean this without affecting the user
+		p.logger.Errorf("failed to prune leftover volumes for %s/%s: %w, resources may be left", pod.Namespace, pod.Name, err)
+	}
+	p.logger.Infof("Deleted pod %s", pod.Name)
+	return nil
+}
+
+// pruneUnusedVolumes removes volumes in use by pod that aren't used by any other pods
+func (p *Provider) pruneUnusedVolumes(ctx context.Context, pod *corev1.Pod) error {
+	rawSecrets, rawConfigMaps := getSecretsAndConfigmaps(pod)
+	// since this pod was removed, originally mark all of the secrets/configmaps it uses as eligible
+	// for pruning
+	pruneSecrets := sets.Set[string]{}.Insert(rawSecrets...)
+	pruneConfigMap := sets.Set[string]{}.Insert(rawConfigMaps...)
+	var pods corev1.PodList
+	// only pods in the same namespace could be using secrets/configmaps that this pod is using
+	err := p.VirtualClient.List(ctx, &pods, &client.ListOptions{
+		Namespace: pod.Namespace,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to list pods: %w", err)
+	}
+	for _, vPod := range pods.Items {
+		if vPod.Name == pod.Name {
+			continue
+		}
+		secrets, configMaps := getSecretsAndConfigmaps(&vPod)
+		pruneSecrets.Delete(secrets...)
+		pruneConfigMap.Delete(configMaps...)
+	}
+	for _, secretName := range pruneSecrets.UnsortedList() {
+		var secret corev1.Secret
+		err := p.VirtualClient.Get(ctx, types.NamespacedName{
+			Name:      secretName,
+			Namespace: pod.Namespace,
+		}, &secret)
+		if err != nil {
+			return fmt.Errorf("unable to get secret %s/%s for pod volume: %w", pod.Namespace, secretName, err)
+		}
+		err = p.Handler.RemoveResource(ctx, &secret)
+		if err != nil {
+			return fmt.Errorf("unable to remove secret %s/%s for pod volume: %w", pod.Namespace, secretName, err)
+		}
+	}
+	for _, configMapName := range pruneConfigMap.UnsortedList() {
+		var configMap corev1.ConfigMap
+		err := p.VirtualClient.Get(ctx, types.NamespacedName{
+			Name:      configMapName,
+			Namespace: pod.Namespace,
+		}, &configMap)
+		if err != nil {
+			return fmt.Errorf("unable to get configMap %s/%s for pod volume: %w", pod.Namespace, configMapName, err)
+		}
+
+		if err = p.Handler.RemoveResource(ctx, &configMap); err != nil {
+			return fmt.Errorf("unable to remove configMap %s/%s for pod volume: %w", pod.Namespace, configMapName, err)
+		}
+	}
+	return nil
+}
+
+// GetPod retrieves a pod by name from the provider (can be cached).
+// The Pod returned is expected to be immutable, and may be accessed
+// concurrently outside of the calling goroutine. Therefore it is recommended
+// to return a version after DeepCopy.
+func (p *Provider) GetPod(ctx context.Context, namespace, name string) (*corev1.Pod, error) {
+	p.logger.Debugw("got a request for get pod", "Namespace", namespace, "Name", name)
+	hostNamespaceName := types.NamespacedName{
+		Namespace: p.ClusterNamespace,
+		Name:      p.Translater.TranslateName(namespace, name),
+	}
+	var pod corev1.Pod
+	err := p.HostClient.Get(ctx, hostNamespaceName, &pod)
+	if err != nil {
+		return nil, fmt.Errorf("error when retrieving pod: %w", err)
+	}
+	p.Translater.TranslateFrom(&pod)
+	return &pod, nil
+}
+
+// GetPodStatus retrieves the status of a pod by name from the provider.
+// The PodStatus returned is expected to be immutable, and may be accessed
+// concurrently outside of the calling goroutine. Therefore it is recommended
+// to return a version after DeepCopy.
+func (p *Provider) GetPodStatus(ctx context.Context, namespace, name string) (*corev1.PodStatus, error) {
+	p.logger.Debugw("got a request for pod status", "Namespace", namespace, "Name", name)
+	pod, err := p.GetPod(ctx, namespace, name)
+	if err != nil {
+		return nil, fmt.Errorf("unable to get pod for status: %w", err)
+	}
+	p.logger.Debugw("got pod status", "Namespace", namespace, "Name", name, "Status", pod.Status)
+	return pod.Status.DeepCopy(), nil
+}
+
+// GetPods retrieves a list of all pods running on the provider (can be cached).
+// The Pods returned are expected to be immutable, and may be accessed
+// concurrently outside of the calling goroutine. Therefore it is recommended
+// to return a version after DeepCopy.
+func (p *Provider) GetPods(ctx context.Context) ([]*corev1.Pod, error) {
+	selector := labels.NewSelector()
+	requirement, err := labels.NewRequirement(translate.ClusterNameLabel, selection.Equals, []string{p.ClusterName})
+	if err != nil {
+		return nil, fmt.Errorf("unable to create label selector: %w", err)
+	}
+	selector = selector.Add(*requirement)
+	var podList corev1.PodList
+	err = p.HostClient.List(ctx, &podList, &client.ListOptions{LabelSelector: selector})
+	if err != nil {
+		return nil, fmt.Errorf("unable to list pods: %w", err)
+	}
+	retPods := []*corev1.Pod{}
+	for _, pod := range podList.DeepCopy().Items {
+		p.Translater.TranslateFrom(&pod)
+		retPods = append(retPods, &pod)
+	}
+	return retPods, nil
+}
+
+func (p *Provider) configureNetworking(podName, podNamespace string, pod *corev1.Pod) {
+	// inject networking information to the pod's environment variables
+	for i := range pod.Spec.Containers {
+		pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env,
+			corev1.EnvVar{
+				Name:  "KUBERNETES_PORT_443_TCP",
+				Value: "tcp://" + p.serverIP + ":6443",
+			},
+			corev1.EnvVar{
+				Name:  "KUBERNETES_PORT",
+				Value: "tcp://" + p.serverIP + ":6443",
+			},
+			corev1.EnvVar{
+				Name:  "KUBERNETES_PORT_443_TCP_ADDR",
+				Value: p.serverIP,
+			},
+			corev1.EnvVar{
+				Name:  "KUBERNETES_SERVICE_HOST",
+				Value: p.serverIP,
+			},
+			corev1.EnvVar{
+				Name:  "KUBERNETES_SERVICE_PORT",
+				Value: "6443",
+			},
+		)
+	}
+	// injecting cluster DNS IP to the pods except for coredns pod
+	if !strings.HasPrefix(podName, "coredns") {
+		pod.Spec.DNSPolicy = corev1.DNSNone
+		pod.Spec.DNSConfig = &corev1.PodDNSConfig{
+			Nameservers: []string{
+				p.dnsIP,
+			},
+			Searches: []string{
+				podNamespace + ".svc.cluster.local", "svc.cluster.local", "cluster.local",
+			},
+		}
+	}
+
+}
+
+// getSecretsAndConfigmaps retrieves a list of all secrets/configmaps that are in use by a given pod. Useful
+// for removing/seeing which virtual cluster resources need to be in the host cluster.
+func getSecretsAndConfigmaps(pod *corev1.Pod) ([]string, []string) {
+	var secrets []string
+	var configMaps []string
+	for _, volume := range pod.Spec.Volumes {
+		if volume.Secret != nil {
+			secrets = append(secrets, volume.Secret.SecretName)
+		} else if volume.ConfigMap != nil {
+			configMaps = append(configMaps, volume.ConfigMap.Name)
+		} else if volume.Projected != nil {
+			for _, source := range volume.Projected.Sources {
+				if source.ConfigMap != nil {
+					configMaps = append(configMaps, source.ConfigMap.Name)
+				} else if source.Secret != nil {
+					secrets = append(secrets, source.Secret.Name)
+				}
+			}
+		}
+	}
+	return secrets, configMaps
+}

k3k-kubelet/provider/token.go

@@ -0,0 +1,118 @@
+package provider
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	k3kcontroller "github.com/rancher/k3k/pkg/controller"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
+)
+
+const (
+	kubeAPIAccessPrefix          = "kube-api-access"
+	serviceAccountTokenMountPath = "/var/run/secrets/kubernetes.io/serviceaccount"
+)
+
+// transformTokens copies the serviceaccount tokens used by pod's serviceaccount to a secret on the host cluster and mount it
+// to look like the serviceaccount token
+func (p *Provider) transformTokens(ctx context.Context, pod, tPod *corev1.Pod) error {
+	p.logger.Infow("transforming token", "Pod", pod.Name, "Namespace", pod.Namespace, "serviceAccountName", pod.Spec.ServiceAccountName)
+
+	virtualSecretName := k3kcontroller.SafeConcatNameWithPrefix(pod.Spec.ServiceAccountName, "token")
+	virtualSecret := virtualSecret(virtualSecretName, pod.Namespace, pod.Spec.ServiceAccountName)
+	if err := p.VirtualClient.Create(ctx, virtualSecret); err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return err
+		}
+	}
+	// extracting the tokens data from the secret we just created
+	virtualSecretKey := types.NamespacedName{
+		Name:      virtualSecret.Name,
+		Namespace: virtualSecret.Namespace,
+	}
+	if err := p.VirtualClient.Get(ctx, virtualSecretKey, virtualSecret); err != nil {
+		return err
+	}
+	// To avoid race conditions we need to check if the secret's data has been populated
+	// including the token, ca.crt and namespace
+	if len(virtualSecret.Data) < 3 {
+		return fmt.Errorf("token secret %s/%s data is empty", virtualSecret.Namespace, virtualSecret.Name)
+	}
+	hostSecret := virtualSecret.DeepCopy()
+	hostSecret.Type = ""
+	hostSecret.Annotations = make(map[string]string)
+	p.Translater.TranslateTo(hostSecret)
+
+	if err := p.HostClient.Create(ctx, hostSecret); err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return err
+		}
+	}
+	p.translateToken(tPod, hostSecret.Name)
+	return nil
+}
+
+func virtualSecret(name, namespace, serviceAccountName string) *corev1.Secret {
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			Annotations: map[string]string{
+				corev1.ServiceAccountNameKey: serviceAccountName,
+			},
+		},
+		Type: corev1.SecretTypeServiceAccountToken,
+	}
+}
+
+// translateToken will remove the serviceaccount from the pod and replace the kube-api-access volume
+// with a custom token volume and mount it to all containers within the pod
+func (p *Provider) translateToken(pod *corev1.Pod, hostSecretName string) {
+	pod.Spec.ServiceAccountName = ""
+	pod.Spec.DeprecatedServiceAccount = ""
+	pod.Spec.AutomountServiceAccountToken = ptr.To(false)
+	removeKubeAccessVolume(pod)
+	addKubeAccessVolume(pod, hostSecretName)
+}
+
+func removeKubeAccessVolume(pod *corev1.Pod) {
+	for i, volume := range pod.Spec.Volumes {
+		if strings.HasPrefix(volume.Name, kubeAPIAccessPrefix) {
+			pod.Spec.Volumes = append(pod.Spec.Volumes[:i], pod.Spec.Volumes[i+1:]...)
+		}
+	}
+	for i, container := range pod.Spec.Containers {
+		for j, mountPath := range container.VolumeMounts {
+			if strings.HasPrefix(mountPath.Name, kubeAPIAccessPrefix) {
+				pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts[:j], pod.Spec.Containers[i].VolumeMounts[j+1:]...)
+			}
+		}
+	}
+}
+
+func addKubeAccessVolume(pod *corev1.Pod, hostSecretName string) {
+	var tokenVolumeName = k3kcontroller.SafeConcatNameWithPrefix(kubeAPIAccessPrefix)
+	pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
+		Name: tokenVolumeName,
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: hostSecretName,
+			},
+		},
+	})
+	for i := range pod.Spec.Containers {
+		pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, corev1.VolumeMount{
+			Name:      tokenVolumeName,
+			MountPath: serviceAccountTokenMountPath,
+		})
+	}
+}

k3k-kubelet/provider/util.go

@@ -0,0 +1,25 @@
+package provider
+
+import (
+	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+// translatorSizeQueue feeds the size events from the WebSocket
+// resizeChan into the SPDY client input. Implements TerminalSizeQueue
+// interface.
+type translatorSizeQueue struct {
+	resizeChan <-chan api.TermSize
+}
+
+func (t *translatorSizeQueue) Next() *remotecommand.TerminalSize {
+	size, ok := <-t.resizeChan
+	if !ok {
+		return nil
+	}
+	newSize := remotecommand.TerminalSize{
+		Width:  size.Width,
+		Height: size.Height,
+	}
+	return &newSize
+}

k3k-kubelet/translate/host.go

@@ -0,0 +1,106 @@
+package translate
+
+import (
+	"encoding/hex"
+	"fmt"
+
+	"github.com/rancher/k3k/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// ClusterNameLabel is the key for the label that contains the name of the virtual cluster
+	// this resource was made in
+	ClusterNameLabel = "k3k.io/clusterName"
+	// ResourceNameAnnotation is the key for the annotation that contains the original name of this
+	// resource in the virtual cluster
+	ResourceNameAnnotation = "k3k.io/name"
+	// ResourceNamespaceAnnotation is the key for the annotation that contains the original namespace of this
+	// resource in the virtual cluster
+	ResourceNamespaceAnnotation = "k3k.io/namespace"
+)
+
+type ToHostTranslater struct {
+	// ClusterName is the name of the virtual cluster whose resources we are
+	// translating to a host cluster
+	ClusterName string
+	// ClusterNamespace is the namespace of the virtual cluster whose resources
+	// we are tranlsating to a host cluster
+	ClusterNamespace string
+}
+
+// Translate translates a virtual cluster object to a host cluster object. This should only be used for
+// static resources such as configmaps/secrets, and not for things like pods (which can reference other
+// objects). Note that this won't set host-cluster values (like resource version) so when updating you
+// may need to fetch the existing value and do some combination before using this.
+func (t *ToHostTranslater) TranslateTo(obj client.Object) {
+	// owning objects may be in the virtual cluster, but may not be in the host cluster
+	obj.SetOwnerReferences(nil)
+	// add some annotations to make it easier to track source object
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	annotations[ResourceNameAnnotation] = obj.GetName()
+	annotations[ResourceNamespaceAnnotation] = obj.GetNamespace()
+	obj.SetAnnotations(annotations)
+	// add a label to quickly identify objects owned by a given virtual cluster
+	labels := obj.GetLabels()
+	if labels == nil {
+		labels = map[string]string{}
+	}
+	labels[ClusterNameLabel] = t.ClusterName
+	obj.SetLabels(labels)
+
+	// resource version/UID won't match what's in the host cluster.
+	obj.SetResourceVersion("")
+	obj.SetUID("")
+
+	// set the name and the namespace so that this goes in the proper host namespace
+	// and doesn't collide with other resources
+	obj.SetName(t.TranslateName(obj.GetNamespace(), obj.GetName()))
+	obj.SetNamespace(t.ClusterNamespace)
+	obj.SetFinalizers(nil)
+}
+
+func (t *ToHostTranslater) TranslateFrom(obj client.Object) {
+	// owning objects may be in the virtual cluster, but may not be in the host cluster
+	obj.SetOwnerReferences(nil)
+
+	// remove the annotations added to track original name
+	annotations := obj.GetAnnotations()
+	// TODO: It's possible that this was erased by a change on the host cluster
+	// In this case, we need to have some sort of fallback or error return
+	name := annotations[ResourceNameAnnotation]
+	namespace := annotations[ResourceNamespaceAnnotation]
+	obj.SetName(name)
+	obj.SetNamespace(namespace)
+	delete(annotations, ResourceNameAnnotation)
+	delete(annotations, ResourceNamespaceAnnotation)
+	obj.SetAnnotations(annotations)
+
+	// remove the clusteName tracking label
+	labels := obj.GetLabels()
+	delete(labels, ClusterNameLabel)
+	obj.SetLabels(labels)
+
+	// resource version/UID won't match what's in the virtual cluster.
+	obj.SetResourceVersion("")
+	obj.SetUID("")
+
+}
+
+// TranslateName returns the name of the resource in the host cluster. Will not update the object with this name.
+func (t *ToHostTranslater) TranslateName(namespace string, name string) string {
+	// we need to come up with a name which is:
+	// - somewhat connectable to the original resource
+	// - a valid k8s name
+	// - idempotently calculatable
+	// - unique for this combination of name/namespace/cluster
+	namePrefix := fmt.Sprintf("%s-%s-%s", name, namespace, t.ClusterName)
+	// use + as a separator since it can't be in an object name
+	nameKey := fmt.Sprintf("%s+%s+%s", name, namespace, t.ClusterName)
+	// it's possible that the suffix will be in the name, so we use hex to make it valid for k8s
+	nameSuffix := hex.EncodeToString([]byte(nameKey))
+	return controller.SafeConcatName(namePrefix, nameSuffix)
+}

main.go

@@ -3,50 +3,129 @@ package main
 
 import (
 	"context"
-	"flag"
+	"fmt"
+	"os"
 
+	"github.com/go-logr/zapr"
+	"github.com/rancher/k3k/cli/cmds"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller/cluster"
+	"github.com/rancher/k3k/pkg/controller/clusterset"
+	"github.com/rancher/k3k/pkg/log"
+	"github.com/urfave/cli"
+	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
-	ctrlconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
+	ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 )
 
-var Scheme = runtime.NewScheme()
+const (
+	program   = "k3k"
+	version   = "dev"
+	gitCommit = "HEAD"
+)
+
+var (
+	scheme           = runtime.NewScheme()
+	clusterCIDR      string
+	sharedAgentImage string
+	kubeconfig       string
+	debug            bool
+	logger           *log.Logger
+	flags            = []cli.Flag{
+		cli.StringFlag{
+			Name:        "kubeconfig",
+			EnvVar:      "KUBECONFIG",
+			Usage:       "Kubeconfig path",
+			Destination: &kubeconfig,
+		},
+		cli.StringFlag{
+			Name:        "cluster-cidr",
+			EnvVar:      "CLUSTER_CIDR",
+			Usage:       "Cluster CIDR to be added to the networkpolicy of the clustersets",
+			Destination: &clusterCIDR,
+		},
+		cli.StringFlag{
+			Name:        "shared-agent-image",
+			EnvVar:      "SHARED_AGENT_IMAGE",
+			Usage:       "K3K Virtual Kubelet image",
+			Value:       "rancher/k3k:k3k-kubelet-dev",
+			Destination: &sharedAgentImage,
+		},
+		cli.BoolFlag{
+			Name:        "debug",
+			EnvVar:      "DEBUG",
+			Usage:       "Debug level logging",
+			Destination: &debug,
+		},
+	}
+)
 
 func init() {
-	_ = clientgoscheme.AddToScheme(Scheme)
-	_ = v1alpha1.AddToScheme(Scheme)
+	_ = clientgoscheme.AddToScheme(scheme)
+	_ = v1alpha1.AddToScheme(scheme)
 }
 
 func main() {
-	ctrlconfig.RegisterFlags(nil)
-	flag.Parse()
+	app := cmds.NewApp()
+	app.Flags = flags
+	app.Action = run
+	app.Version = version + " (" + gitCommit + ")"
+	app.Before = func(clx *cli.Context) error {
+		logger = log.New(debug)
+		return nil
+	}
+	if err := app.Run(os.Args); err != nil {
+		logger.Fatalw("failed to run k3k controller", zap.Error(err))
+	}
 
+}
+
+func run(clx *cli.Context) error {
 	ctx := context.Background()
 
-	kubeconfig := flag.Lookup("kubeconfig").Value.String()
 	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
 	if err != nil {
-		klog.Fatalf("Failed to create config from kubeconfig file: %v", err)
+		return fmt.Errorf("failed to create config from kubeconfig file: %v", err)
 	}
 
 	mgr, err := ctrl.NewManager(restConfig, manager.Options{
-		Scheme: Scheme,
+		Scheme: scheme,
 	})
+
 	if err != nil {
-		klog.Fatalf("Failed to create new controller runtime manager: %v", err)
+		return fmt.Errorf("failed to create new controller runtime manager: %v", err)
 	}
 
-	if err := cluster.Add(ctx, mgr); err != nil {
-		klog.Fatalf("Failed to add the new controller: %v", err)
+	ctrlruntimelog.SetLogger(zapr.NewLogger(logger.Desugar().WithOptions(zap.AddCallerSkip(1))))
+	logger.Info("adding cluster controller")
+	if err := cluster.Add(ctx, mgr, sharedAgentImage, logger); err != nil {
+		return fmt.Errorf("failed to add the new cluster controller: %v", err)
+	}
+
+	logger.Info("adding etcd pod controller")
+	if err := cluster.AddPodController(ctx, mgr, logger); err != nil {
+		return fmt.Errorf("failed to add the new cluster controller: %v", err)
+	}
+
+	logger.Info("adding clusterset controller")
+	if err := clusterset.Add(ctx, mgr, clusterCIDR, logger); err != nil {
+		return fmt.Errorf("failed to add the clusterset controller: %v", err)
+	}
+
+	if clusterCIDR == "" {
+		logger.Info("adding networkpolicy node controller")
+		if err := clusterset.AddNodeController(ctx, mgr, logger); err != nil {
+			return fmt.Errorf("failed to add the clusterset node controller: %v", err)
+		}
 	}
 
 	if err := mgr.Start(ctx); err != nil {
-		klog.Fatalf("Failed to start the manager: %v", err)
+		return fmt.Errorf("failed to start the manager: %v", err)
 	}
+
+	return nil
 }

ops/build

@@ -19,17 +19,28 @@ if [ "$CROSS" = "true" ] && [ "$ARCH" = "amd64" ]; then
     CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o bin/k3k-arm64 
     GOOS=freebsd GOARCH=arm64 go build -ldflags "$LINKFLAGS" -o bin/k3k-freebsd
     GOOS=darwin GOARCH=amd64 go build -ldflags "$LINKFLAGS" -o bin/k3k-darwin-amd64 
-    GOOS=darwin GOARCH=arm64 go build -ldflags "$LINKFLAGS" -o bin/k3k-darwin
+    GOOS=darwin GOARCH=arm64 go build -ldflags "$LINKFLAGS" -o bin/k3k-darwin-aarch64
     GOOS=windows GOARCH=amd64 go build -ldflags "$LINKFLAGS" -o bin/k3k-windows
 fi
 
+# build k3k-kubelet
+CGO_ENABLED=0 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o bin/k3k-kubelet ./k3k-kubelet
+if [ "$CROSS" = "true" ] && [ "$ARCH" = "amd64" ]; then
+    CGO_ENABLED=0 GOOS=linux GOARCH=s390x go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o bin/k3k-kubelet-s390x 
+    CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o bin/k3k-kubelet-arm64 
+    GOOS=freebsd GOARCH=arm64 go build -ldflags "$LINKFLAGS" -o bin/k3k-kubelet-freebsd
+    GOOS=darwin GOARCH=amd64 go build -ldflags "$LINKFLAGS" -o bin/k3k-kubelet-darwin-amd64 
+    GOOS=darwin GOARCH=arm64 go build -ldflags "$LINKFLAGS" -o bin/k3k-kubelet-darwin-aarch64
+    GOOS=windows GOARCH=amd64 go build -ldflags "$LINKFLAGS" -o bin/k3k-kubelet-windows
+fi
+
 # build k3kcli
 CGO_ENABLED=0 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o bin/k3kcli ./cli
 if [ "$CROSS" = "true" ] && [ "$ARCH" = "amd64" ]; then
     CGO_ENABLED=0 GOOS=linux GOARCH=s390x go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o bin/k3kcli-s390x ./cli
     CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o bin/k3kcli-arm64 ./cli
-    GOOS=freebsd GOARCH=arm64 go build -ldflags "$LINKFLAGS" -o bin/k3k-freebsd ./cli
-    GOOS=darwin GOARCH=amd64 go build -ldflags "$LINKFLAGS" -o bin/k3kcli-darwin-adm64 ./cli
-    GOOS=darwin GOARCH=arm64 go build -ldflags "$LINKFLAGS" -o bin/k3kcli-darwin ./cli
+    GOOS=freebsd GOARCH=arm64 go build -ldflags "$LINKFLAGS" -o bin/k3kcli-freebsd ./cli
+    GOOS=darwin GOARCH=amd64 go build -ldflags "$LINKFLAGS" -o bin/k3kcli-darwin-amd64 ./cli
+    GOOS=darwin GOARCH=arm64 go build -ldflags "$LINKFLAGS" -o bin/k3kcli-darwin-aarch64 ./cli
     GOOS=windows GOARCH=amd64 go build -ldflags "$LINKFLAGS" -o bin/k3kcli-windows ./cli
 fi

ops/build-crds

@@ -0,0 +1,8 @@
+#! /bin/sh
+
+cd $(dirname $0)/../
+
+# This will return non-zero until all of our objects in ./pkg/apis can generate valid crds.
+# allowDangerousTypes is needed for struct that use floats
+controller-gen crd:generateEmbeddedObjectMeta=true,allowDangerousTypes=false paths=./pkg/apis/... output:crd:dir=./charts/k3k/crds
+

ops/checksum

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -ex
+
+cd $(dirname $0)/..
+
+CHECKSUM_DIR=${CHECKSUM_DIR:-./bin}
+
+sumfile="${CHECKSUM_DIR}/sha256sum.txt"
+echo -n "" > "${sumfile}"
+
+files=$(ls ${CHECKSUM_DIR} | grep -v "sha256sum.txt")
+for file in ${files}; do
+  sha256sum "${CHECKSUM_DIR}/${file}" | sed "s;$(dirname ${CHECKSUM_DIR}/${file})/;;g" >> "${sumfile}"
+done
+
+cat "${sumfile}"

ops/ci

@@ -4,6 +4,7 @@ set -e
 cd $(dirname $0)
 
 ./build
+./checksum
 ./test
 ./validate
 ./validate-ci

ops/package

@@ -8,6 +8,7 @@ cd $(dirname $0)/..
 mkdir -p dist/artifacts
 cp bin/k3k dist/artifacts/k3k${SUFFIX}
 cp bin/k3kcli dist/artifacts/k3kcli${SUFFIX}
+cp bin/k3k-kubelet dist/artifacts/k3k-kubelet${SUFFIX}
 
 IMAGE=${REPO}/k3k:${TAG}
 DOCKERFILE=package/Dockerfile
@@ -17,3 +18,13 @@ fi
 
 docker build -f ${DOCKERFILE} -t ${IMAGE} .
 echo Built ${IMAGE}
+
+# todo: This might need to go to it's own repo
+IMAGE=${REPO}/k3k:${TAG}-kubelet
+DOCKERFILE=package/Dockerfile.kubelet
+if [ -e ${DOCKERFILE}.${ARCH} ]; then
+    DOCKERFILE=${DOCKERFILE}.${ARCH}
+fi
+
+docker build -f ${DOCKERFILE} -t ${IMAGE} .
+echo Built ${IMAGE}

ops/version

@@ -5,7 +5,7 @@ if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
 fi
 
 COMMIT=$(git rev-parse --short HEAD)
-GIT_TAG=${DRONE_TAG:-$(git tag -l --contains HEAD | head -n 1)}
+GIT_TAG=${TAG:-$(git tag -l --contains HEAD | head -n 1)}
 
 if [[ -z "$DIRTY" && -n "$GIT_TAG" ]]; then
     VERSION=$GIT_TAG
@@ -19,9 +19,15 @@ fi
 
 SUFFIX="-${ARCH}"
 
-TAG=${TAG:-${VERSION}${SUFFIX}}
+
+if [[ $VERSION = "chart*" ]]; then
+  TAG=${TAG:-${VERSION}}
+else 
+  TAG=${TAG:-${VERSION}${SUFFIX}}
+fi
+
 REPO=${REPO:-rancher}
 
-if echo $TAG | grep -q dirty; then
+if echo $TAG | grep dirty; then
     TAG=dev
 fi

package/Dockerfile

@@ -1,4 +1,9 @@
 FROM alpine
-COPY bin/k3k /usr/bin/
-COPY bin/k3kcli /usr/bin/
+
+ARG BIN_K3K=bin/k3k
+ARG BIN_K3KCLI=bin/k3kcli
+
+COPY ${BIN_K3K} /usr/bin/
+COPY ${BIN_K3KCLI} /usr/bin/
+
 CMD ["k3k"]

package/Dockerfile.kubelet

@@ -0,0 +1,8 @@
+# TODO: swicth this to BCI-micro or scratch. Left as base right now so that debug can be done a bit easier
+FROM registry.suse.com/bci/bci-base:15.6
+
+ARG BIN_K3K_KUBELET=bin/k3k-kubelet
+
+COPY ${BIN_K3K_KUBELET} /usr/bin/
+
+ENTRYPOINT ["/usr/bin/k3k-kubelet"]

pkg/apis/k3k.io/v1alpha1/register.go

@@ -21,7 +21,10 @@ func Resource(resource string) schema.GroupResource {
 func addKnownTypes(s *runtime.Scheme) error {
 	s.AddKnownTypes(SchemeGroupVersion,
 		&Cluster{},
-		&ClusterList{})
+		&ClusterList{},
+		&ClusterSet{},
+		&ClusterSetList{},
+	)
 	metav1.AddToGroupVersion(s, SchemeGroupVersion)
 	return nil
 }

pkg/apis/k3k.io/v1alpha1/set_types.go

@@ -0,0 +1,86 @@
+package v1alpha1
+
+import (
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +genclient
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:storageversion
+// +kubebuilder:subresource:status
+
+type ClusterSet struct {
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	metav1.TypeMeta   `json:",inline"`
+
+	// +kubebuilder:default={}
+	//
+	// Spec is the spec of the ClusterSet
+	Spec ClusterSetSpec `json:"spec"`
+
+	// Status is the status of the ClusterSet
+	Status ClusterSetStatus `json:"status,omitempty"`
+}
+
+type ClusterSetSpec struct {
+	// MaxLimits are the limits that apply to all clusters (server + agent) in the set
+	MaxLimits v1.ResourceList `json:"maxLimits,omitempty"`
+
+	// DefaultLimits are the limits used for servers/agents when a cluster in the set doesn't provide any
+	DefaultLimits *ClusterLimit `json:"defaultLimits,omitempty"`
+
+	// DefaultNodeSelector is the node selector that applies to all clusters (server + agent) in the set
+	DefaultNodeSelector map[string]string `json:"defaultNodeSelector,omitempty"`
+
+	// DefaultPriorityClass is the priorityClassName applied to all pods of all clusters in the set
+	DefaultPriorityClass string `json:"defaultPriorityClass,omitempty"`
+
+	// DisableNetworkPolicy is an option that will disable the creation of a default networkpolicy for cluster isolation
+	DisableNetworkPolicy bool `json:"disableNetworkPolicy,omitempty"`
+
+	// +kubebuilder:default={shared}
+	// +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf"
+	// +kubebuilder:validation:MinItems=1
+	//
+	// AllowedNodeTypes are the allowed cluster provisioning modes. Defaults to [shared].
+	AllowedNodeTypes []ClusterMode `json:"allowedNodeTypes,omitempty"`
+
+	// PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace.
+	PodSecurityAdmissionLevel *PodSecurityAdmissionLevel `json:"podSecurityAdmissionLevel,omitempty"`
+}
+
+// +kubebuilder:validation:Enum=privileged;baseline;restricted
+//
+// PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace.
+type PodSecurityAdmissionLevel string
+
+const (
+	PrivilegedPodSecurityAdmissionLevel = PodSecurityAdmissionLevel("privileged")
+	BaselinePodSecurityAdmissionLevel   = PodSecurityAdmissionLevel("baseline")
+	RestrictedPodSecurityAdmissionLevel = PodSecurityAdmissionLevel("restricted")
+)
+
+type ClusterSetStatus struct {
+
+	// ObservedGeneration was the generation at the time the status was updated.
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// LastUpdate is the timestamp when the status was last updated
+	LastUpdate string `json:"lastUpdateTime,omitempty"`
+
+	// Summary is a summary of the status
+	Summary string `json:"summary,omitempty"`
+
+	// Conditions are the invidual conditions for the cluster set
+	Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+type ClusterSetList struct {
+	metav1.ListMeta `json:"metadata,omitempty"`
+	metav1.TypeMeta `json:",inline"`
+
+	Items []ClusterSet `json:"items"`
+}

pkg/apis/k3k.io/v1alpha1/types.go

@@ -1,36 +1,109 @@
 package v1alpha1
 
 import (
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:storageversion
+// +kubebuilder:subresource:status
 
 type Cluster struct {
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 	metav1.TypeMeta   `json:",inline"`
 
 	Spec   ClusterSpec   `json:"spec"`
-	Status ClusterStatus `json:"status"`
+	Status ClusterStatus `json:"status,omitempty"`
 }
 
 type ClusterSpec struct {
-	Name        string   `json:"name"`
-	Version     string   `json:"version"`
-	Servers     *int32   `json:"servers"`
-	Agents      *int32   `json:"agents"`
-	Token       string   `json:"token"`
-	ClusterCIDR string   `json:"clusterCIDR,omitempty"`
-	ServiceCIDR string   `json:"serviceCIDR,omitempty"`
-	ClusterDNS  string   `json:"clusterDNS,omitempty"`
-	ServerArgs  []string `json:"serverArgs,omitempty"`
-	AgentArgs   []string `json:"agentArgs,omitempty"`
-	TLSSANs     []string `json:"tlsSANs,omitempty"`
-	Addons      []Addon  `json:"addons,omitempty"`
+	// Version is a string representing the Kubernetes version to be used by the virtual nodes.
+	Version string `json:"version"`
 
+	// Servers is the number of K3s pods to run in server (controlplane) mode.
+	// +kubebuilder:validation:XValidation:message="cluster must have at least one server",rule="self >= 1"
+	Servers *int32 `json:"servers"`
+
+	// Agents is the number of K3s pods to run in agent (worker) mode.
+	// +kubebuilder:validation:XValidation:message="invalid value for agents",rule="self >= 0"
+	Agents *int32 `json:"agents"`
+
+	// NodeSelector is the node selector that will be applied to all server/agent pods.
+	// In "shared" mode the node selector will be applied also to the workloads.
+	// +optional
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+
+	// PriorityClass is the priorityClassName that will be applied to all server/agent pods.
+	// In "shared" mode the priorityClassName will be applied also to the workloads.
+	PriorityClass string `json:"priorityClass,omitempty"`
+
+	// Limit is the limits that apply for the server/worker nodes.
+	Limit *ClusterLimit `json:"clusterLimit,omitempty"`
+
+	// TokenSecretRef is Secret reference used as a token join server and worker nodes to the cluster. The controller
+	// assumes that the secret has a field "token" in its data, any other fields in the secret will be ignored.
+	// +optional
+	TokenSecretRef *v1.SecretReference `json:"tokenSecretRef"`
+
+	// ClusterCIDR is the CIDR range for the pods of the cluster. Defaults to 10.42.0.0/16.
+	// +kubebuilder:validation:XValidation:message="clusterCIDR is immutable",rule="self == oldSelf"
+	ClusterCIDR string `json:"clusterCIDR,omitempty"`
+
+	// ServiceCIDR is the CIDR range for the services in the cluster. Defaults to 10.43.0.0/16.
+	// +kubebuilder:validation:XValidation:message="serviceCIDR is immutable",rule="self == oldSelf"
+	ServiceCIDR string `json:"serviceCIDR,omitempty"`
+
+	// ClusterDNS is the IP address for the coredns service. Needs to be in the range provided by ServiceCIDR or CoreDNS may not deploy.
+	// Defaults to 10.43.0.10.
+	// +kubebuilder:validation:XValidation:message="clusterDNS is immutable",rule="self == oldSelf"
+	ClusterDNS string `json:"clusterDNS,omitempty"`
+
+	// ServerArgs are the ordered key value pairs (e.x. "testArg", "testValue") for the K3s pods running in server mode.
+	ServerArgs []string `json:"serverArgs,omitempty"`
+
+	// AgentArgs are the ordered key value pairs (e.x. "testArg", "testValue") for the K3s pods running in agent mode.
+	AgentArgs []string `json:"agentArgs,omitempty"`
+
+	// TLSSANs are the subjectAlternativeNames for the certificate the K3s server will use.
+	TLSSANs []string `json:"tlsSANs,omitempty"`
+
+	// Addons is a list of secrets containing raw YAML which will be deployed in the virtual K3k cluster on startup.
+	Addons []Addon `json:"addons,omitempty"`
+
+	// Mode is the cluster provisioning mode which can be either "shared" or "virtual". Defaults to "shared"
+	// +kubebuilder:default="shared"
+	// +kubebuilder:validation:Enum=shared;virtual
+	// +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf"
+	Mode ClusterMode `json:"mode"`
+
+	// Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data
+	// persistence is guaranteed, so restart of a virtual cluster pod may result in data loss without this field.
 	Persistence *PersistenceConfig `json:"persistence,omitempty"`
-	Expose      *ExposeConfig      `json:"expose,omitempty"`
+
+	// Expose contains options for exposing the apiserver inside/outside of the cluster. By default, this is only exposed as a
+	// clusterIP which is relatively secure, but difficult to access outside of the cluster.
+	// +optional
+	Expose *ExposeConfig `json:"expose,omitempty"`
+}
+
+// +kubebuilder:validation:Enum=shared;virtual
+// +kubebuilder:default="shared"
+//
+// ClusterMode is the possible provisioning mode of a Cluster.
+type ClusterMode string
+
+const (
+	SharedClusterMode  = ClusterMode("shared")
+	VirtualClusterMode = ClusterMode("virtual")
+)
+
+type ClusterLimit struct {
+	// ServerLimit is the limits (cpu/mem) that apply to the server nodes
+	ServerLimit v1.ResourceList `json:"serverLimit,omitempty"`
+	// WorkerLimit is the limits (cpu/mem) that apply to the agent nodes
+	WorkerLimit v1.ResourceList `json:"workerLimit,omitempty"`
 }
 
 type Addon struct {
@@ -49,20 +122,24 @@ type ClusterList struct {
 
 type PersistenceConfig struct {
 	// Type can be ephermal, static, dynamic
+	// +kubebuilder:default="ephemeral"
 	Type               string `json:"type"`
 	StorageClassName   string `json:"storageClassName,omitempty"`
 	StorageRequestSize string `json:"storageRequestSize,omitempty"`
 }
 
 type ExposeConfig struct {
-	Ingress      *IngressConfig      `json:"ingress"`
-	LoadBalancer *LoadBalancerConfig `json:"loadbalancer"`
-	NodePort     *NodePortConfig     `json:"nodePort"`
+	// +optional
+	Ingress *IngressConfig `json:"ingress,omitempty"`
+	// +optional
+	LoadBalancer *LoadBalancerConfig `json:"loadbalancer,omitempty"`
+	// +optional
+	NodePort *NodePortConfig `json:"nodePort,omitempty"`
 }
 
 type IngressConfig struct {
-	Enabled          bool   `json:"enabled"`
-	IngressClassName string `json:"ingressClassName"`
+	Enabled          bool   `json:"enabled,omitempty"`
+	IngressClassName string `json:"ingressClassName,omitempty"`
 }
 
 type LoadBalancerConfig struct {

pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go

@@ -6,6 +6,8 @@
 package v1alpha1
 
 import (
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -53,6 +55,36 @@ func (in *Cluster) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClusterLimit) DeepCopyInto(out *ClusterLimit) {
+	*out = *in
+	if in.ServerLimit != nil {
+		in, out := &in.ServerLimit, &out.ServerLimit
+		*out = make(v1.ResourceList, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val.DeepCopy()
+		}
+	}
+	if in.WorkerLimit != nil {
+		in, out := &in.WorkerLimit, &out.WorkerLimit
+		*out = make(v1.ResourceList, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val.DeepCopy()
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterLimit.
+func (in *ClusterLimit) DeepCopy() *ClusterLimit {
+	if in == nil {
+		return nil
+	}
+	out := new(ClusterLimit)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClusterList) DeepCopyInto(out *ClusterList) {
 	*out = *in
@@ -86,6 +118,125 @@ func (in *ClusterList) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClusterSet) DeepCopyInto(out *ClusterSet) {
+	*out = *in
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	out.TypeMeta = in.TypeMeta
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSet.
+func (in *ClusterSet) DeepCopy() *ClusterSet {
+	if in == nil {
+		return nil
+	}
+	out := new(ClusterSet)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ClusterSet) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClusterSetList) DeepCopyInto(out *ClusterSetList) {
+	*out = *in
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	out.TypeMeta = in.TypeMeta
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]ClusterSet, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSetList.
+func (in *ClusterSetList) DeepCopy() *ClusterSetList {
+	if in == nil {
+		return nil
+	}
+	out := new(ClusterSetList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ClusterSetList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClusterSetSpec) DeepCopyInto(out *ClusterSetSpec) {
+	*out = *in
+	if in.MaxLimits != nil {
+		in, out := &in.MaxLimits, &out.MaxLimits
+		*out = make(v1.ResourceList, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val.DeepCopy()
+		}
+	}
+	if in.DefaultLimits != nil {
+		in, out := &in.DefaultLimits, &out.DefaultLimits
+		*out = new(ClusterLimit)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.DefaultNodeSelector != nil {
+		in, out := &in.DefaultNodeSelector, &out.DefaultNodeSelector
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSetSpec.
+func (in *ClusterSetSpec) DeepCopy() *ClusterSetSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ClusterSetSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClusterSetStatus) DeepCopyInto(out *ClusterSetStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]metav1.Condition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSetStatus.
+func (in *ClusterSetStatus) DeepCopy() *ClusterSetStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ClusterSetStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 	*out = *in
@@ -99,6 +250,18 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.NodeSelector != nil {
+		in, out := &in.NodeSelector, &out.NodeSelector
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Limit != nil {
+		in, out := &in.Limit, &out.Limit
+		*out = new(ClusterLimit)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.ServerArgs != nil {
 		in, out := &in.ServerArgs, &out.ServerArgs
 		*out = make([]string, len(*in))

pkg/controller/certs/certs.go

@@ -0,0 +1,71 @@
+package certs
+
+import (
+	"crypto"
+	"crypto/x509"
+	"fmt"
+	"net"
+	"time"
+
+	certutil "github.com/rancher/dynamiclistener/cert"
+)
+
+func CreateClientCertKey(commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, expiresAt time.Duration, caCert, caKey string) ([]byte, []byte, error) {
+	caKeyPEM, err := certutil.ParsePrivateKeyPEM([]byte(caKey))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caCertPEM, err := certutil.ParseCertsPEM([]byte(caCert))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	b, err := generateKey()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	key, err := certutil.ParsePrivateKeyPEM(b)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfg := certutil.Config{
+		CommonName:   commonName,
+		Organization: organization,
+		Usages:       extKeyUsage,
+		ExpiresAt:    expiresAt,
+	}
+	if altNames != nil {
+		cfg.AltNames = *altNames
+	}
+	cert, err := certutil.NewSignedCert(cfg, key.(crypto.Signer), caCertPEM[0], caKeyPEM.(crypto.Signer))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCertPEM[0])...), b, nil
+}
+
+func generateKey() (data []byte, err error) {
+	generatedData, err := certutil.MakeEllipticPrivateKeyPEM()
+	if err != nil {
+		return nil, fmt.Errorf("error generating key: %v", err)
+	}
+
+	return generatedData, nil
+}
+
+func AddSANs(sans []string) certutil.AltNames {
+	var altNames certutil.AltNames
+	for _, san := range sans {
+		ip := net.ParseIP(san)
+		if ip == nil {
+			altNames.DNSNames = append(altNames.DNSNames, san)
+		} else {
+			altNames.IPs = append(altNames.IPs, ip)
+		}
+	}
+	return altNames
+}

pkg/controller/cluster/agent/agent.go

@@ -2,245 +2,27 @@ package agent
 
 import (
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	"github.com/rancher/k3k/pkg/controller/util"
-	apps "k8s.io/api/apps/v1"
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/utils/pointer"
+	"github.com/rancher/k3k/pkg/controller"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-const agentName = "k3k-agent"
+const (
+	configName = "agent-config"
+)
 
-type Agent struct {
-	cluster *v1alpha1.Cluster
+type Agent interface {
+	Name() string
+	Config() ctrlruntimeclient.Object
+	Resources() ([]ctrlruntimeclient.Object, error)
 }
 
-func New(cluster *v1alpha1.Cluster) *Agent {
-	return &Agent{
-		cluster: cluster,
+func New(cluster *v1alpha1.Cluster, serviceIP, sharedAgentImage, token string) Agent {
+	if cluster.Spec.Mode == VirtualNodeMode {
+		return NewVirtualAgent(cluster, serviceIP, token)
 	}
+	return NewSharedAgent(cluster, serviceIP, sharedAgentImage, token)
 }
 
-func (a *Agent) Deploy() *apps.Deployment {
-	image := util.K3SImage(a.cluster)
-
-	const name = "k3k-agent"
-
-	return &apps.Deployment{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Deployment",
-			APIVersion: "apps/v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      a.cluster.Name + "-" + name,
-			Namespace: util.ClusterNamespace(a.cluster),
-		},
-		Spec: apps.DeploymentSpec{
-			Replicas: a.cluster.Spec.Agents,
-			Selector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{
-					"cluster": a.cluster.Name,
-					"type":    "agent",
-				},
-			},
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						"cluster": a.cluster.Name,
-						"type":    "agent",
-					},
-				},
-				Spec: a.podSpec(image, name, a.cluster.Spec.AgentArgs, false),
-			},
-		},
-	}
-}
-
-func (a *Agent) StatefulAgent(cluster *v1alpha1.Cluster) *apps.StatefulSet {
-	image := util.K3SImage(cluster)
-
-	return &apps.StatefulSet{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Statefulset",
-			APIVersion: "apps/v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      cluster.Name + "-" + agentName,
-			Namespace: util.ClusterNamespace(cluster),
-		},
-		Spec: apps.StatefulSetSpec{
-			ServiceName: cluster.Name + "-" + agentName + "-headless",
-			Replicas:    cluster.Spec.Agents,
-			Selector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{
-					"cluster": cluster.Name,
-					"type":    "agent",
-				},
-			},
-			VolumeClaimTemplates: []v1.PersistentVolumeClaim{
-				{
-					TypeMeta: metav1.TypeMeta{
-						Kind:       "PersistentVolumeClaim",
-						APIVersion: "v1",
-					},
-					ObjectMeta: metav1.ObjectMeta{
-						Name:      "varlibrancherk3s",
-						Namespace: util.ClusterNamespace(cluster),
-					},
-					Spec: v1.PersistentVolumeClaimSpec{
-						AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
-						StorageClassName: &cluster.Status.Persistence.StorageClassName,
-						Resources: v1.ResourceRequirements{
-							Requests: v1.ResourceList{
-								"storage": resource.MustParse(cluster.Status.Persistence.StorageRequestSize),
-							},
-						},
-					},
-				},
-				{
-					TypeMeta: metav1.TypeMeta{
-						Kind:       "PersistentVolumeClaim",
-						APIVersion: "v1",
-					},
-					ObjectMeta: metav1.ObjectMeta{
-						Name:      "varlibkubelet",
-						Namespace: util.ClusterNamespace(cluster),
-					},
-					Spec: v1.PersistentVolumeClaimSpec{
-						Resources: v1.ResourceRequirements{
-							Requests: v1.ResourceList{
-								"storage": resource.MustParse(cluster.Status.Persistence.StorageRequestSize),
-							},
-						},
-						AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
-						StorageClassName: &cluster.Status.Persistence.StorageClassName,
-					},
-				},
-			},
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						"cluster": cluster.Name,
-						"type":    "agent",
-					},
-				},
-				Spec: a.podSpec(image, agentName, cluster.Spec.AgentArgs, true),
-			},
-		},
-	}
-}
-
-func (a *Agent) podSpec(image, name string, args []string, statefulSet bool) v1.PodSpec {
-	args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...)
-	podSpec := v1.PodSpec{
-		Volumes: []v1.Volume{
-			{
-				Name: "config",
-				VolumeSource: v1.VolumeSource{
-					Secret: &v1.SecretVolumeSource{
-						SecretName: name + "-config",
-						Items: []v1.KeyToPath{
-							{
-								Key:  "config.yaml",
-								Path: "config.yaml",
-							},
-						},
-					},
-				},
-			},
-			{
-				Name: "run",
-				VolumeSource: v1.VolumeSource{
-					EmptyDir: &v1.EmptyDirVolumeSource{},
-				},
-			},
-			{
-				Name: "varrun",
-				VolumeSource: v1.VolumeSource{
-					EmptyDir: &v1.EmptyDirVolumeSource{},
-				},
-			},
-			{
-				Name: "varlibcni",
-				VolumeSource: v1.VolumeSource{
-					EmptyDir: &v1.EmptyDirVolumeSource{},
-				},
-			},
-			{
-				Name: "varlog",
-				VolumeSource: v1.VolumeSource{
-					EmptyDir: &v1.EmptyDirVolumeSource{},
-				},
-			},
-		},
-		Containers: []v1.Container{
-			{
-				Name:  name,
-				Image: image,
-				SecurityContext: &v1.SecurityContext{
-					Privileged: pointer.Bool(true),
-				},
-				Command: []string{
-					"/bin/k3s",
-				},
-				Args: args,
-				VolumeMounts: []v1.VolumeMount{
-					{
-						Name:      "config",
-						MountPath: "/opt/rancher/k3s/",
-						ReadOnly:  false,
-					},
-					{
-						Name:      "run",
-						MountPath: "/run",
-						ReadOnly:  false,
-					},
-					{
-						Name:      "varrun",
-						MountPath: "/var/run",
-						ReadOnly:  false,
-					},
-					{
-						Name:      "varlibcni",
-						MountPath: "/var/lib/cni",
-						ReadOnly:  false,
-					},
-					{
-						Name:      "varlibkubelet",
-						MountPath: "/var/lib/kubelet",
-						ReadOnly:  false,
-					},
-					{
-						Name:      "varlibrancherk3s",
-						MountPath: "/var/lib/rancher/k3s",
-						ReadOnly:  false,
-					},
-					{
-						Name:      "varlog",
-						MountPath: "/var/log",
-						ReadOnly:  false,
-					},
-				},
-			},
-		},
-	}
-
-	if !statefulSet {
-		podSpec.Volumes = append(podSpec.Volumes, v1.Volume{
-			Name: "varlibkubelet",
-			VolumeSource: v1.VolumeSource{
-				EmptyDir: &v1.EmptyDirVolumeSource{},
-			},
-		}, v1.Volume{
-
-			Name: "varlibrancherk3s",
-			VolumeSource: v1.VolumeSource{
-				EmptyDir: &v1.EmptyDirVolumeSource{},
-			},
-		},
-		)
-	}
-
-	return podSpec
+func configSecretName(clusterName string) string {
+	return controller.SafeConcatNameWithPrefix(clusterName, configName)
 }

pkg/controller/cluster/agent/service.go

@@ -1,30 +0,0 @@
-package agent
-
-import (
-	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	"github.com/rancher/k3k/pkg/controller/util"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func (a *Agent) StatefulAgentService(cluster *v1alpha1.Cluster) *v1.Service {
-	return &v1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      cluster.Name + "-" + agentName + "-headless",
-			Namespace: util.ClusterNamespace(cluster),
-		},
-		Spec: v1.ServiceSpec{
-			Type:      v1.ServiceTypeClusterIP,
-			ClusterIP: v1.ClusterIPNone,
-			Selector: map[string]string{
-				"cluster": cluster.Name,
-				"role":    "agent",
-			},
-			Ports: []v1.ServicePort{},
-		},
-	}
-}

pkg/controller/cluster/agent/shared.go

@@ -0,0 +1,404 @@
+package agent
+
+import (
+	"crypto"
+	"crypto/x509"
+	"fmt"
+	"time"
+
+	certutil "github.com/rancher/dynamiclistener/cert"
+	"github.com/rancher/k3k/k3k-kubelet/translate"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/certs"
+	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	sharedKubeletConfigPath = "/opt/rancher/k3k/config.yaml"
+	SharedNodeAgentName     = "kubelet"
+	SharedNodeMode          = "shared"
+)
+
+type SharedAgent struct {
+	cluster          *v1alpha1.Cluster
+	serviceIP        string
+	sharedAgentImage string
+	token            string
+}
+
+func NewSharedAgent(cluster *v1alpha1.Cluster, serviceIP, sharedAgentImage, token string) Agent {
+	return &SharedAgent{
+		cluster:          cluster,
+		serviceIP:        serviceIP,
+		sharedAgentImage: sharedAgentImage,
+		token:            token,
+	}
+}
+
+func (s *SharedAgent) Config() ctrlruntimeclient.Object {
+	config := sharedAgentData(s.cluster, s.token, s.Name(), s.serviceIP)
+
+	return &v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configSecretName(s.cluster.Name),
+			Namespace: s.cluster.Namespace,
+		},
+		Data: map[string][]byte{
+			"config.yaml": []byte(config),
+		},
+	}
+}
+
+func sharedAgentData(cluster *v1alpha1.Cluster, token, nodeName, ip string) string {
+	return fmt.Sprintf(`clusterName: %s
+clusterNamespace: %s
+nodeName: %s
+agentHostname: %s
+serverIP: %s
+token: %s`,
+		cluster.Name, cluster.Namespace, nodeName, nodeName, ip, token)
+}
+
+func (s *SharedAgent) Resources() ([]ctrlruntimeclient.Object, error) {
+	// generate certs for webhook
+	certSecret, err := s.webhookTLS()
+	if err != nil {
+		return nil, err
+	}
+	return []ctrlruntimeclient.Object{
+		s.serviceAccount(),
+		s.role(),
+		s.roleBinding(),
+		s.service(),
+		s.deployment(),
+		s.dnsService(),
+		certSecret}, nil
+}
+
+func (s *SharedAgent) deployment() *apps.Deployment {
+	selector := &metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"cluster": s.cluster.Name,
+			"type":    "agent",
+			"mode":    "shared",
+		},
+	}
+
+	return &apps.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      s.Name(),
+			Namespace: s.cluster.Namespace,
+			Labels:    selector.MatchLabels,
+		},
+		Spec: apps.DeploymentSpec{
+			Selector: selector,
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: selector.MatchLabels,
+				},
+				Spec: s.podSpec(selector),
+			},
+		},
+	}
+}
+
+func (s *SharedAgent) podSpec(affinitySelector *metav1.LabelSelector) v1.PodSpec {
+	var limit v1.ResourceList
+
+	return v1.PodSpec{
+		Affinity: &v1.Affinity{
+			PodAntiAffinity: &v1.PodAntiAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
+					{
+						LabelSelector: affinitySelector,
+						TopologyKey:   "kubernetes.io/hostname",
+					},
+				},
+			},
+		},
+		ServiceAccountName: s.Name(),
+		Volumes: []v1.Volume{
+			{
+				Name: "config",
+				VolumeSource: v1.VolumeSource{
+					Secret: &v1.SecretVolumeSource{
+						SecretName: configSecretName(s.cluster.Name),
+						Items: []v1.KeyToPath{
+							{
+								Key:  "config.yaml",
+								Path: "config.yaml",
+							},
+						},
+					},
+				},
+			},
+			{
+				Name: "webhook-certs",
+				VolumeSource: v1.VolumeSource{
+					Secret: &v1.SecretVolumeSource{
+						SecretName: WebhookSecretName(s.cluster.Name),
+						Items: []v1.KeyToPath{
+							{
+								Key:  "tls.crt",
+								Path: "tls.crt",
+							},
+							{
+								Key:  "tls.key",
+								Path: "tls.key",
+							},
+							{
+								Key:  "ca.crt",
+								Path: "ca.crt",
+							},
+						},
+					},
+				},
+			},
+		},
+		Containers: []v1.Container{
+			{
+				Name:            s.Name(),
+				Image:           s.sharedAgentImage,
+				ImagePullPolicy: v1.PullAlways,
+				Resources: v1.ResourceRequirements{
+					Limits: limit,
+				},
+				Args: []string{
+					"--config",
+					sharedKubeletConfigPath,
+				},
+				VolumeMounts: []v1.VolumeMount{
+					{
+						Name:      "config",
+						MountPath: "/opt/rancher/k3k/",
+						ReadOnly:  false,
+					},
+					{
+						Name:      "webhook-certs",
+						MountPath: "/opt/rancher/k3k-webhook",
+						ReadOnly:  false,
+					},
+				},
+				Ports: []v1.ContainerPort{
+					{
+						Name:          "webhook-port",
+						Protocol:      v1.ProtocolTCP,
+						ContainerPort: 9443,
+					},
+				},
+			},
+		}}
+}
+
+func (s *SharedAgent) service() *v1.Service {
+	return &v1.Service{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      s.Name(),
+			Namespace: s.cluster.Namespace,
+		},
+		Spec: v1.ServiceSpec{
+			Type: v1.ServiceTypeClusterIP,
+			Selector: map[string]string{
+				"cluster": s.cluster.Name,
+				"type":    "agent",
+				"mode":    "shared",
+			},
+			Ports: []v1.ServicePort{
+				{
+					Name:     "k3s-kubelet-port",
+					Protocol: v1.ProtocolTCP,
+					Port:     10250,
+				},
+				{
+					Name:       "webhook-server",
+					Protocol:   v1.ProtocolTCP,
+					Port:       9443,
+					TargetPort: intstr.FromInt32(9443),
+				},
+			},
+		},
+	}
+}
+
+func (s *SharedAgent) dnsService() *v1.Service {
+	return &v1.Service{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      s.DNSName(),
+			Namespace: s.cluster.Namespace,
+		},
+		Spec: v1.ServiceSpec{
+			Type: v1.ServiceTypeClusterIP,
+			Selector: map[string]string{
+				translate.ClusterNameLabel: s.cluster.Name,
+				"k8s-app":                  "kube-dns",
+			},
+			Ports: []v1.ServicePort{
+				{
+					Name:       "dns",
+					Protocol:   v1.ProtocolUDP,
+					Port:       53,
+					TargetPort: intstr.FromInt32(53),
+				},
+				{
+					Name:       "dns-tcp",
+					Protocol:   v1.ProtocolTCP,
+					Port:       53,
+					TargetPort: intstr.FromInt32(53),
+				},
+				{
+					Name:       "metrics",
+					Protocol:   v1.ProtocolTCP,
+					Port:       9153,
+					TargetPort: intstr.FromInt32(9153),
+				},
+			},
+		},
+	}
+}
+
+func (s *SharedAgent) serviceAccount() *v1.ServiceAccount {
+	return &v1.ServiceAccount{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ServiceAccount",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      s.Name(),
+			Namespace: s.cluster.Namespace,
+		},
+	}
+}
+
+func (s *SharedAgent) role() *rbacv1.Role {
+	return &rbacv1.Role{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Role",
+			APIVersion: "rbac.authorization.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      s.Name(),
+			Namespace: s.cluster.Namespace,
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"persistentvolumeclaims", "pods", "pods/log", "pods/exec", "secrets", "configmaps", "services"},
+				Verbs:     []string{"*"},
+			},
+			{
+				APIGroups: []string{"k3k.io"},
+				Resources: []string{"clusters"},
+				Verbs:     []string{"get", "watch", "list"},
+			},
+		},
+	}
+}
+
+func (s *SharedAgent) roleBinding() *rbacv1.RoleBinding {
+	return &rbacv1.RoleBinding{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "RoleBinding",
+			APIVersion: "rbac.authorization.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      s.Name(),
+			Namespace: s.cluster.Namespace,
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     s.Name(),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      s.Name(),
+				Namespace: s.cluster.Namespace,
+			},
+		},
+	}
+}
+
+func (s *SharedAgent) Name() string {
+	return controller.SafeConcatNameWithPrefix(s.cluster.Name, SharedNodeAgentName)
+}
+
+func (s *SharedAgent) DNSName() string {
+	return controller.SafeConcatNameWithPrefix(s.cluster.Name, "kube-dns")
+}
+
+func (s *SharedAgent) webhookTLS() (*v1.Secret, error) {
+	// generate CA CERT/KEY
+	caKeyBytes, err := certutil.MakeEllipticPrivateKeyPEM()
+	if err != nil {
+		return nil, err
+	}
+
+	caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := certutil.Config{
+		CommonName: fmt.Sprintf("k3k-webhook-ca@%d", time.Now().Unix()),
+	}
+
+	caCert, err := certutil.NewSelfSignedCACert(cfg, caKey.(crypto.Signer))
+	if err != nil {
+		return nil, err
+	}
+
+	caCertBytes := certutil.EncodeCertPEM(caCert)
+	// generate webhook cert bundle
+	altNames := certs.AddSANs([]string{s.Name(), s.cluster.Name})
+	webhookCert, webhookKey, err := certs.CreateClientCertKey(
+		s.Name(), nil,
+		&altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, time.Hour*24*time.Duration(356),
+		string(caCertBytes),
+		string(caKeyBytes))
+	if err != nil {
+		return nil, err
+	}
+	return &v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      WebhookSecretName(s.cluster.Name),
+			Namespace: s.cluster.Namespace,
+		},
+		Data: map[string][]byte{
+			"tls.crt": webhookCert,
+			"tls.key": webhookKey,
+			"ca.crt":  caCertBytes,
+			"ca.key":  caKeyBytes,
+		},
+	}, nil
+}
+
+func WebhookSecretName(clusterName string) string {
+	return controller.SafeConcatNameWithPrefix(clusterName, "webhook")
+}

pkg/controller/cluster/agent/virtual.go

@@ -0,0 +1,223 @@
+package agent
+
+import (
+	"fmt"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
+	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	VirtualNodeMode      = "virtual"
+	virtualNodeAgentName = "agent"
+)
+
+type VirtualAgent struct {
+	cluster   *v1alpha1.Cluster
+	serviceIP string
+	token     string
+}
+
+func NewVirtualAgent(cluster *v1alpha1.Cluster, serviceIP, token string) Agent {
+	return &VirtualAgent{
+		cluster:   cluster,
+		serviceIP: serviceIP,
+		token:     token,
+	}
+}
+
+func (v *VirtualAgent) Config() ctrlruntimeclient.Object {
+	config := virtualAgentData(v.serviceIP, v.token)
+
+	return &v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configSecretName(v.cluster.Name),
+			Namespace: v.cluster.Namespace,
+		},
+		Data: map[string][]byte{
+			"config.yaml": []byte(config),
+		},
+	}
+}
+
+func (v *VirtualAgent) Resources() ([]ctrlruntimeclient.Object, error) {
+	return []ctrlruntimeclient.Object{v.deployment()}, nil
+}
+
+func virtualAgentData(serviceIP, token string) string {
+	return fmt.Sprintf(`server: https://%s:6443
+token: %s
+with-node-id: true`, serviceIP, token)
+}
+
+func (v *VirtualAgent) deployment() *apps.Deployment {
+	image := controller.K3SImage(v.cluster)
+
+	const name = "k3k-agent"
+	selector := metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"cluster": v.cluster.Name,
+			"type":    "agent",
+			"mode":    "virtual",
+		},
+	}
+	return &apps.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      v.Name(),
+			Namespace: v.cluster.Namespace,
+			Labels:    selector.MatchLabels,
+		},
+		Spec: apps.DeploymentSpec{
+			Replicas: v.cluster.Spec.Agents,
+			Selector: &selector,
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: selector.MatchLabels,
+				},
+				Spec: v.podSpec(image, name, v.cluster.Spec.AgentArgs, &selector),
+			},
+		},
+	}
+}
+
+func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelector *metav1.LabelSelector) v1.PodSpec {
+	var limit v1.ResourceList
+	args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...)
+	podSpec := v1.PodSpec{
+		Affinity: &v1.Affinity{
+			PodAntiAffinity: &v1.PodAntiAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
+					{
+						LabelSelector: affinitySelector,
+						TopologyKey:   "kubernetes.io/hostname",
+					},
+				},
+			},
+		},
+		Volumes: []v1.Volume{
+			{
+				Name: "config",
+				VolumeSource: v1.VolumeSource{
+					Secret: &v1.SecretVolumeSource{
+						SecretName: configSecretName(v.cluster.Name),
+						Items: []v1.KeyToPath{
+							{
+								Key:  "config.yaml",
+								Path: "config.yaml",
+							},
+						},
+					},
+				},
+			},
+			{
+				Name: "run",
+				VolumeSource: v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: "varrun",
+				VolumeSource: v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: "varlibcni",
+				VolumeSource: v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: "varlog",
+				VolumeSource: v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: "varlibkubelet",
+				VolumeSource: v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: "varlibrancherk3s",
+				VolumeSource: v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{},
+				},
+			},
+		},
+		Containers: []v1.Container{
+			{
+				Name:            name,
+				Image:           image,
+				ImagePullPolicy: v1.PullAlways,
+				SecurityContext: &v1.SecurityContext{
+					Privileged: ptr.To(true),
+				},
+				Args: args,
+				Command: []string{
+					"/bin/k3s",
+				},
+				Resources: v1.ResourceRequirements{
+					Limits: limit,
+				},
+				VolumeMounts: []v1.VolumeMount{
+					{
+						Name:      "config",
+						MountPath: "/opt/rancher/k3s/",
+						ReadOnly:  false,
+					},
+					{
+						Name:      "run",
+						MountPath: "/run",
+						ReadOnly:  false,
+					},
+					{
+						Name:      "varrun",
+						MountPath: "/var/run",
+						ReadOnly:  false,
+					},
+					{
+						Name:      "varlibcni",
+						MountPath: "/var/lib/cni",
+						ReadOnly:  false,
+					},
+					{
+						Name:      "varlibkubelet",
+						MountPath: "/var/lib/kubelet",
+						ReadOnly:  false,
+					},
+					{
+						Name:      "varlibrancherk3s",
+						MountPath: "/var/lib/rancher/k3s",
+						ReadOnly:  false,
+					},
+					{
+						Name:      "varlog",
+						MountPath: "/var/log",
+						ReadOnly:  false,
+					},
+				},
+			},
+		},
+	}
+
+	return podSpec
+}
+
+func (v *VirtualAgent) Name() string {
+	return controller.SafeConcatNameWithPrefix(v.cluster.Name, virtualNodeAgentName)
+}

pkg/controller/cluster/cluster.go

@@ -2,41 +2,34 @@ package cluster
 
 import (
 	"context"
-	"crypto/tls"
-	"crypto/x509"
 	"errors"
 	"fmt"
-	"net/url"
-	"strings"
+	"reflect"
 	"time"
 
-	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
 	"github.com/rancher/k3k/pkg/controller/cluster/agent"
-	"github.com/rancher/k3k/pkg/controller/cluster/config"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
-	"github.com/rancher/k3k/pkg/controller/util"
-	"github.com/sirupsen/logrus"
-	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
-	clientv3 "go.etcd.io/etcd/client/v3"
-	apps "k8s.io/api/apps/v1"
+	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	"github.com/rancher/k3k/pkg/log"
+	"go.uber.org/zap"
 	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/util/retry"
-	"k8s.io/klog"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	ctrlruntimecontroller "sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
 const (
+	namePrefix           = "k3k"
 	clusterController    = "k3k-cluster-controller"
 	clusterFinalizerName = "cluster.k3k.io/finalizer"
 	etcdPodFinalizerName = "etcdpod.k3k.io/finalizer"
@@ -51,139 +44,91 @@ const (
 )
 
 type ClusterReconciler struct {
-	Client client.Client
-	Scheme *runtime.Scheme
+	Client           ctrlruntimeclient.Client
+	Scheme           *runtime.Scheme
+	SharedAgentImage string
+	logger           *log.Logger
 }
 
 // Add adds a new controller to the manager
-func Add(ctx context.Context, mgr manager.Manager) error {
+func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage string, logger *log.Logger) error {
 	// initialize a new Reconciler
 	reconciler := ClusterReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:           mgr.GetClient(),
+		Scheme:           mgr.GetScheme(),
+		SharedAgentImage: sharedAgentImage,
+		logger:           logger.Named(clusterController),
 	}
-
-	// create a new controller and add it to the manager
-	//this can be replaced by the new builder functionality in controller-runtime
-	controller, err := controller.New(clusterController, mgr, controller.Options{
-		Reconciler:              &reconciler,
-		MaxConcurrentReconciles: maxConcurrentReconciles,
-	})
-	if err != nil {
-		return err
-	}
-
-	if err := controller.Watch(&source.Kind{Type: &v1alpha1.Cluster{}}, &handler.EnqueueRequestForObject{}); err != nil {
-		return err
-	}
-	return controller.Watch(&source.Kind{Type: &v1.Pod{}},
-		&handler.EnqueueRequestForOwner{IsController: true, OwnerType: &apps.StatefulSet{}})
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.Cluster{}).
+		WithOptions(ctrlruntimecontroller.Options{
+			MaxConcurrentReconciles: maxConcurrentReconciles,
+		}).
+		Complete(&reconciler)
 }
 
 func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
-
 	var (
-		cluster     v1alpha1.Cluster
-		podList     v1.PodList
-		clusterName string
+		cluster v1alpha1.Cluster
+		podList v1.PodList
 	)
-	if req.Namespace != "" {
-		s := strings.Split(req.Namespace, "-")
-		if len(s) <= 1 {
-			return reconcile.Result{}, util.LogAndReturnErr("failed to get cluster namespace", nil)
-		}
-
-		clusterName = s[1]
-		var cluster v1alpha1.Cluster
-		if err := c.Client.Get(ctx, types.NamespacedName{Name: clusterName}, &cluster); err != nil {
-			return reconcile.Result{}, util.LogAndReturnErr("failed to get cluster object", err)
-		}
-		if *cluster.Spec.Servers == 1 {
-			klog.Infof("skipping request for etcd pod for cluster [%s] since it is not in HA mode", clusterName)
-			return reconcile.Result{}, nil
-		}
-		matchingLabels := client.MatchingLabels(map[string]string{"role": "server"})
-		listOpts := &client.ListOptions{Namespace: req.Namespace}
-		matchingLabels.ApplyToList(listOpts)
-
-		if err := c.Client.List(ctx, &podList, listOpts); err != nil {
-			return reconcile.Result{}, client.IgnoreNotFound(err)
-		}
-		for _, pod := range podList.Items {
-			klog.Infof("Handle etcd server pod [%s/%s]", pod.Namespace, pod.Name)
-			if err := c.handleServerPod(ctx, cluster, &pod); err != nil {
-				return reconcile.Result{}, util.LogAndReturnErr("failed to handle etcd pod", err)
-			}
-		}
-
-		return reconcile.Result{}, nil
-	}
-
+	log := c.logger.With("Cluster", req.NamespacedName)
 	if err := c.Client.Get(ctx, req.NamespacedName, &cluster); err != nil {
-		return reconcile.Result{}, client.IgnoreNotFound(err)
+		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
 	}
-
 	if cluster.DeletionTimestamp.IsZero() {
 		if !controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) {
 			controllerutil.AddFinalizer(&cluster, clusterFinalizerName)
 			if err := c.Client.Update(ctx, &cluster); err != nil {
-				return reconcile.Result{}, util.LogAndReturnErr("failed to add cluster finalizer", err)
+				return reconcile.Result{}, err
 			}
 		}
-
-		// we create a namespace for each new cluster
-		var ns v1.Namespace
-		objKey := client.ObjectKey{
-			Name: util.ClusterNamespace(&cluster),
-		}
-		if err := c.Client.Get(ctx, objKey, &ns); err != nil {
-			if !apierrors.IsNotFound(err) {
-				return reconcile.Result{}, util.LogAndReturnErr("failed to get cluster namespace "+util.ClusterNamespace(&cluster), err)
-			}
-		}
-
-		klog.Infof("enqueue cluster [%s]", cluster.Name)
-		if err := c.createCluster(ctx, &cluster); err != nil {
-			return reconcile.Result{}, util.LogAndReturnErr("failed to create cluster", err)
-		}
-		return reconcile.Result{}, nil
+		log.Info("enqueue cluster")
+		return reconcile.Result{}, c.createCluster(ctx, &cluster, log)
 	}
 
 	// remove finalizer from the server pods and update them.
-	matchingLabels := client.MatchingLabels(map[string]string{"role": "server"})
-	listOpts := &client.ListOptions{Namespace: util.ClusterNamespace(&cluster)}
+	matchingLabels := ctrlruntimeclient.MatchingLabels(map[string]string{"role": "server"})
+	listOpts := &ctrlruntimeclient.ListOptions{Namespace: cluster.Namespace}
 	matchingLabels.ApplyToList(listOpts)
-
 	if err := c.Client.List(ctx, &podList, listOpts); err != nil {
-		return reconcile.Result{}, client.IgnoreNotFound(err)
+		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
 	}
 	for _, pod := range podList.Items {
 		if controllerutil.ContainsFinalizer(&pod, etcdPodFinalizerName) {
 			controllerutil.RemoveFinalizer(&pod, etcdPodFinalizerName)
 			if err := c.Client.Update(ctx, &pod); err != nil {
-				return reconcile.Result{}, util.LogAndReturnErr("failed to remove etcd finalizer", err)
+				return reconcile.Result{}, err
 			}
 		}
 	}
 
+	if err := c.unbindNodeProxyClusterRole(ctx, &cluster); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	if controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) {
 		// remove finalizer from the cluster and update it.
 		controllerutil.RemoveFinalizer(&cluster, clusterFinalizerName)
 		if err := c.Client.Update(ctx, &cluster); err != nil {
-			return reconcile.Result{}, util.LogAndReturnErr("failed to remove cluster finalizer", err)
+			return reconcile.Result{}, err
 		}
 	}
-	klog.Infof("deleting cluster [%s]", cluster.Name)
-
+	log.Info("deleting cluster")
 	return reconcile.Result{}, nil
 }
 
-func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1.Cluster) error {
-	if cluster.Name == ClusterInvalidName {
-		klog.Errorf("Invalid cluster name %s, no action will be taken", cluster.Name)
+func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1.Cluster, log *zap.SugaredLogger) error {
+	if err := c.validate(cluster); err != nil {
+		log.Errorw("invalid change", zap.Error(err))
 		return nil
 	}
-	s := server.New(cluster, c.Client)
+	token, err := c.token(ctx, cluster)
+	if err != nil {
+		return err
+	}
+
+	s := server.New(cluster, c.Client, token, string(cluster.Spec.Mode))
 
 	if cluster.Spec.Persistence != nil {
 		cluster.Status.Persistence = cluster.Spec.Persistence
@@ -192,13 +137,6 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
 			cluster.Status.Persistence.StorageRequestSize = defaultStoragePersistentSize
 		}
 	}
-	if err := c.Client.Update(ctx, cluster); err != nil {
-		return util.LogAndReturnErr("failed to update cluster with persistence type", err)
-	}
-	// create a new namespace for the cluster
-	if err := c.createNamespace(ctx, cluster); err != nil {
-		return util.LogAndReturnErr("failed to create ns", err)
-	}
 
 	cluster.Status.ClusterCIDR = cluster.Spec.ClusterCIDR
 	if cluster.Status.ClusterCIDR == "" {
@@ -210,77 +148,61 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
 		cluster.Status.ServiceCIDR = defaultClusterServiceCIDR
 	}
 
-	klog.Infof("creating cluster service")
+	log.Info("creating cluster service")
 	serviceIP, err := c.createClusterService(ctx, cluster, s)
 	if err != nil {
-		return util.LogAndReturnErr("failed to create cluster service", err)
+		return err
 	}
 
-	if err := c.createClusterConfigs(ctx, cluster, serviceIP); err != nil {
-		return util.LogAndReturnErr("failed to create cluster configs", err)
+	if err := c.createClusterConfigs(ctx, cluster, s, serviceIP); err != nil {
+		return err
 	}
 
 	// creating statefulsets in case the user chose a persistence type other than ephermal
 	if err := c.server(ctx, cluster, s); err != nil {
-		return util.LogAndReturnErr("failed to create servers", err)
+		return err
 	}
 
-	if err := c.agent(ctx, cluster); err != nil {
-		return util.LogAndReturnErr("failed to create agents", err)
+	if err := c.agent(ctx, cluster, serviceIP, token); err != nil {
+		return err
 	}
 
 	if cluster.Spec.Expose != nil {
 		if cluster.Spec.Expose.Ingress != nil {
 			serverIngress, err := s.Ingress(ctx, c.Client)
 			if err != nil {
-				return util.LogAndReturnErr("failed to create ingress object", err)
+				return err
 			}
 
 			if err := c.Client.Create(ctx, serverIngress); err != nil {
 				if !apierrors.IsAlreadyExists(err) {
-					return util.LogAndReturnErr("failed to create server ingress", err)
+					return err
 				}
 			}
 		}
 	}
 
-	kubeconfigSecret, err := s.GenerateNewKubeConfig(ctx, serviceIP)
+	bootstrapSecret, err := bootstrap.Generate(ctx, cluster, serviceIP, token)
 	if err != nil {
-		return util.LogAndReturnErr("failed to generate new kubeconfig", err)
+		return err
 	}
 
-	if err := c.Client.Create(ctx, kubeconfigSecret); err != nil {
+	if err := c.Client.Create(ctx, bootstrapSecret); err != nil {
 		if !apierrors.IsAlreadyExists(err) {
-			return util.LogAndReturnErr("failed to create kubeconfig secret", err)
+			return err
 		}
 	}
 
+	if err := c.bindNodeProxyClusterRole(ctx, cluster); err != nil {
+		return err
+	}
+
 	return c.Client.Update(ctx, cluster)
 }
 
-func (c *ClusterReconciler) createNamespace(ctx context.Context, cluster *v1alpha1.Cluster) error {
-	// create a new namespace for the cluster
-	namespace := v1.Namespace{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: util.ClusterNamespace(cluster),
-		},
-	}
-	if err := controllerutil.SetControllerReference(cluster, &namespace, c.Scheme); err != nil {
-		return err
-	}
-
-	if err := c.Client.Create(ctx, &namespace); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
-			return util.LogAndReturnErr("failed to create ns", err)
-		}
-	}
-
-	return nil
-}
-
-func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP string) error {
+func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server, serviceIP string) error {
 	// create init node config
-	initServerConfig, err := config.Server(cluster, true, serviceIP)
+	initServerConfig, err := server.Config(true, serviceIP)
 	if err != nil {
 		return err
 	}
@@ -296,7 +218,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
 	}
 
 	// create servers configuration
-	serverConfig, err := config.Server(cluster, false, serviceIP)
+	serverConfig, err := server.Config(false, serviceIP)
 	if err != nil {
 		return err
 	}
@@ -309,23 +231,12 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
 		}
 	}
 
-	// create agents configuration
-	agentsConfig := agentConfig(cluster, serviceIP)
-	if err := controllerutil.SetControllerReference(cluster, &agentsConfig, c.Scheme); err != nil {
-		return err
-	}
-	if err := c.Client.Create(ctx, &agentsConfig); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
-			return err
-		}
-	}
-
 	return nil
 }
 
-func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server) (string, error) {
+func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v1alpha1.Cluster, s *server.Server) (string, error) {
 	// create cluster service
-	clusterService := server.Service(cluster)
+	clusterService := s.Service(cluster)
 
 	if err := controllerutil.SetControllerReference(cluster, clusterService, c.Scheme); err != nil {
 		return "", err
@@ -338,9 +249,9 @@ func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v
 
 	var service v1.Service
 
-	objKey := client.ObjectKey{
-		Namespace: util.ClusterNamespace(cluster),
-		Name:      "k3k-server-service",
+	objKey := ctrlruntimeclient.ObjectKey{
+		Namespace: cluster.Namespace,
+		Name:      server.ServiceName(cluster.Name),
 	}
 	if err := c.Client.Get(ctx, objKey, &service); err != nil {
 		return "", err
@@ -351,7 +262,7 @@ func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v
 
 func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server) error {
 	// create headless service for the statefulset
-	serverStatefulService := server.StatefulServerService(cluster)
+	serverStatefulService := server.StatefulServerService()
 	if err := controllerutil.SetControllerReference(cluster, serverStatefulService, c.Scheme); err != nil {
 		return err
 	}
@@ -360,7 +271,7 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluste
 			return err
 		}
 	}
-	ServerStatefulSet, err := server.StatefulServer(ctx, cluster)
+	ServerStatefulSet, err := server.StatefulServer(ctx)
 	if err != nil {
 		return err
 	}
@@ -369,206 +280,128 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluste
 		return err
 	}
 
-	if err := c.Client.Create(ctx, ServerStatefulSet); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *ClusterReconciler) agent(ctx context.Context, cluster *v1alpha1.Cluster) error {
-	agent := agent.New(cluster)
-
-	agentsDeployment := agent.Deploy()
-	if err := controllerutil.SetControllerReference(cluster, agentsDeployment, c.Scheme); err != nil {
+	if err := c.ensure(ctx, ServerStatefulSet, false); err != nil {
 		return err
 	}
 
-	if err := c.Client.Create(ctx, agentsDeployment); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
+	return nil
+}
+
+func (c *ClusterReconciler) bindNodeProxyClusterRole(ctx context.Context, cluster *v1alpha1.Cluster) error {
+	clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
+	if err := c.Client.Get(ctx, types.NamespacedName{Name: "k3k-node-proxy"}, clusterRoleBinding); err != nil {
+		return fmt.Errorf("failed to get or find k3k-node-proxy ClusterRoleBinding: %w", err)
+	}
+
+	subjectName := controller.SafeConcatNameWithPrefix(cluster.Name, agent.SharedNodeAgentName)
+
+	found := false
+	for _, subject := range clusterRoleBinding.Subjects {
+		if subject.Name == subjectName && subject.Namespace == cluster.Namespace {
+			found = true
+		}
+	}
+
+	if !found {
+		clusterRoleBinding.Subjects = append(clusterRoleBinding.Subjects, rbacv1.Subject{
+			Kind:      "ServiceAccount",
+			Name:      subjectName,
+			Namespace: cluster.Namespace,
+		})
+	}
+
+	return c.Client.Update(ctx, clusterRoleBinding)
+}
+
+func (c *ClusterReconciler) unbindNodeProxyClusterRole(ctx context.Context, cluster *v1alpha1.Cluster) error {
+	clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
+	if err := c.Client.Get(ctx, types.NamespacedName{Name: "k3k-node-proxy"}, clusterRoleBinding); err != nil {
+		return fmt.Errorf("failed to get or find k3k-node-proxy ClusterRoleBinding: %w", err)
+	}
+
+	subjectName := controller.SafeConcatNameWithPrefix(cluster.Name, agent.SharedNodeAgentName)
+
+	var cleanedSubjects []rbacv1.Subject
+	for _, subject := range clusterRoleBinding.Subjects {
+		if subject.Name != subjectName || subject.Namespace != cluster.Namespace {
+			cleanedSubjects = append(cleanedSubjects, subject)
+		}
+	}
+
+	// if no subject was removed, all good
+	if reflect.DeepEqual(clusterRoleBinding.Subjects, cleanedSubjects) {
+		return nil
+	}
+
+	clusterRoleBinding.Subjects = cleanedSubjects
+	return c.Client.Update(ctx, clusterRoleBinding)
+}
+
+func (c *ClusterReconciler) agent(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP, token string) error {
+	agent := agent.New(cluster, serviceIP, c.SharedAgentImage, token)
+	agentsConfig := agent.Config()
+	agentResources, err := agent.Resources()
+	if err != nil {
+		return err
+	}
+	agentResources = append(agentResources, agentsConfig)
+
+	return c.ensureAll(ctx, cluster, agentResources)
+}
+
+func (c *ClusterReconciler) validate(cluster *v1alpha1.Cluster) error {
+	if cluster.Name == ClusterInvalidName {
+		return errors.New("invalid cluster name " + cluster.Name + " no action will be taken")
+	}
+	return nil
+}
+
+func (c *ClusterReconciler) ensureAll(ctx context.Context, cluster *v1alpha1.Cluster, objs []ctrlruntimeclient.Object) error {
+	for _, obj := range objs {
+		if err := controllerutil.SetControllerReference(cluster, obj, c.Scheme); err != nil {
+			return err
+		}
+		if err := c.ensure(ctx, obj, false); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func serverData(serviceIP string, cluster *v1alpha1.Cluster) string {
-	return "cluster-init: true\nserver: https://" + serviceIP + ":6443" + serverOptions(cluster)
-}
-
-func initConfigData(cluster *v1alpha1.Cluster) string {
-	return "cluster-init: true\n" + serverOptions(cluster)
-}
-
-func serverOptions(cluster *v1alpha1.Cluster) string {
-	var opts string
-
-	// TODO: generate token if not found
-	if cluster.Spec.Token != "" {
-		opts = "token: " + cluster.Spec.Token + "\n"
-	}
-	if cluster.Status.ClusterCIDR != "" {
-		opts = opts + "cluster-cidr: " + cluster.Status.ClusterCIDR + "\n"
-	}
-	if cluster.Status.ServiceCIDR != "" {
-		opts = opts + "service-cidr: " + cluster.Status.ServiceCIDR + "\n"
-	}
-	if cluster.Spec.ClusterDNS != "" {
-		opts = opts + "cluster-dns: " + cluster.Spec.ClusterDNS + "\n"
-	}
-	if len(cluster.Spec.TLSSANs) > 0 {
-		opts = opts + "tls-san:\n"
-		for _, addr := range cluster.Spec.TLSSANs {
-			opts = opts + "- " + addr + "\n"
+func (c *ClusterReconciler) ensure(ctx context.Context, obj ctrlruntimeclient.Object, requiresRecreate bool) error {
+	exists := true
+	existingObject := obj.DeepCopyObject().(ctrlruntimeclient.Object)
+	if err := c.Client.Get(ctx, types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}, existingObject); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return fmt.Errorf("failed to get Object(%T): %w", existingObject, err)
 		}
+		exists = false
 	}
-	// TODO: Add extra args to the options
 
-	return opts
-}
-
-func agentConfig(cluster *v1alpha1.Cluster, serviceIP string) v1.Secret {
-	config := agentData(serviceIP, cluster.Spec.Token)
-
-	return v1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Secret",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "k3k-agent-config",
-			Namespace: util.ClusterNamespace(cluster),
-		},
-		Data: map[string][]byte{
-			"config.yaml": []byte(config),
-		},
+	if !exists {
+		// if not exists create object
+		if err := c.Client.Create(ctx, obj); err != nil {
+			return err
+		}
+		return nil
+	}
+	// if exists then apply udpate or recreate if necessary
+	if reflect.DeepEqual(obj.(metav1.Object), existingObject.(metav1.Object)) {
+		return nil
 	}
-}
 
-func agentData(serviceIP, token string) string {
-	return fmt.Sprintf(`server: https://%s:6443
-token: %s`, serviceIP, token)
-}
-
-func (c *ClusterReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cluster, pod *v1.Pod) error {
-	if _, ok := pod.Labels["role"]; ok {
-		if pod.Labels["role"] != "server" {
-			return nil
+	if !requiresRecreate {
+		if err := c.Client.Update(ctx, obj); err != nil {
+			return err
 		}
 	} else {
-		return errors.New("server pod has no role label")
-	}
-	// if etcd pod is marked for deletion then we need to remove it from the etcd member list before deletion
-	if !pod.DeletionTimestamp.IsZero() {
-		if cluster.Status.Persistence.Type != server.EphermalNodesType {
-			if controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) {
-				controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName)
-				if err := c.Client.Update(ctx, pod); err != nil {
-					return err
-				}
-			}
-		}
-		tlsConfig, err := c.getETCDTLS(&cluster)
-		if err != nil {
+		// this handles object that needs recreation including configmaps and secrets
+		if err := c.Client.Delete(ctx, obj); err != nil {
 			return err
 		}
-		// remove server from etcd
-		client, err := clientv3.New(clientv3.Config{
-			Endpoints: []string{
-				"https://k3k-server-service." + pod.Namespace + ":2379",
-			},
-			TLS: tlsConfig,
-		})
-		if err != nil {
+		if err := c.Client.Create(ctx, obj); err != nil {
 			return err
 		}
-
-		if err := removePeer(ctx, client, pod.Name, pod.Status.PodIP); err != nil {
-			return err
-		}
-		// remove our finalizer from the list and update it.
-		if controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) {
-			controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName)
-			if err := c.Client.Update(ctx, pod); err != nil {
-				return err
-			}
-		}
 	}
-	if !controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) {
-		controllerutil.AddFinalizer(pod, etcdPodFinalizerName)
-		return c.Client.Update(ctx, pod)
-	}
-
 	return nil
 }
-
-// removePeer removes a peer from the cluster. The peer name and IP address must both match.
-func removePeer(ctx context.Context, client *clientv3.Client, name, address string) error {
-	ctx, cancel := context.WithTimeout(ctx, memberRemovalTimeout)
-	defer cancel()
-	members, err := client.MemberList(ctx)
-	if err != nil {
-		return err
-	}
-
-	for _, member := range members.Members {
-		if !strings.Contains(member.Name, name) {
-			continue
-		}
-		for _, peerURL := range member.PeerURLs {
-			u, err := url.Parse(peerURL)
-			if err != nil {
-				return err
-			}
-			if u.Hostname() == address {
-				logrus.Infof("Removing name=%s id=%d address=%s from etcd", member.Name, member.ID, address)
-				_, err := client.MemberRemove(ctx, member.ID)
-				if errors.Is(err, rpctypes.ErrGRPCMemberNotFound) {
-					return nil
-				}
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (c *ClusterReconciler) getETCDTLS(cluster *v1alpha1.Cluster) (*tls.Config, error) {
-	klog.Infof("generating etcd TLS client certificate for cluster [%s]", cluster.Name)
-	token := cluster.Spec.Token
-	endpoint := "k3k-server-service." + util.ClusterNamespace(cluster)
-	var bootstrap *server.ControlRuntimeBootstrap
-	if err := retry.OnError(retry.DefaultBackoff, func(err error) bool {
-		return true
-	}, func() error {
-		var err error
-		bootstrap, err = server.DecodedBootstrap(token, endpoint)
-		return err
-	}); err != nil {
-		return nil, err
-	}
-
-	etcdCert, etcdKey, err := server.CreateClientCertKey("etcd-client", nil, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, bootstrap.ETCDServerCA.Content, bootstrap.ETCDServerCAKey.Content)
-	if err != nil {
-		return nil, err
-	}
-	clientCert, err := tls.X509KeyPair(etcdCert, etcdKey)
-	if err != nil {
-		return nil, err
-	}
-	// create rootCA CertPool
-	cert, err := certutil.ParseCertsPEM([]byte(bootstrap.ETCDServerCA.Content))
-	if err != nil {
-		return nil, err
-	}
-	pool := x509.NewCertPool()
-	pool.AddCert(cert[0])
-
-	return &tls.Config{
-		RootCAs:      pool,
-		Certificates: []tls.Certificate{clientCert},
-	}, nil
-}

pkg/controller/cluster/config/agent.go

@@ -1,34 +0,0 @@
-package config
-
-import (
-	"fmt"
-
-	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	"github.com/rancher/k3k/pkg/controller/util"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func Agent(cluster *v1alpha1.Cluster, serviceIP string) v1.Secret {
-	config := agentData(serviceIP, cluster.Spec.Token)
-
-	return v1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Secret",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "k3k-agent-config",
-			Namespace: util.ClusterNamespace(cluster),
-		},
-		Data: map[string][]byte{
-			"config.yaml": []byte(config),
-		},
-	}
-}
-
-func agentData(serviceIP, token string) string {
-	return fmt.Sprintf(`server: https://%s:6443
-token: %s
-with-node-id: true`, serviceIP, token)
-}

pkg/controller/cluster/config/server.go

@@ -1,74 +0,0 @@
-package config
-
-import (
-	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	"github.com/rancher/k3k/pkg/controller/util"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func Server(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret, error) {
-	name := "k3k-server-config"
-	if init {
-		name = "k3k-init-server-config"
-	}
-
-	cluster.Status.TLSSANs = append(cluster.Spec.TLSSANs,
-		serviceIP,
-		"k3k-server-service",
-		"k3k-server-service."+util.ClusterNamespace(cluster),
-	)
-
-	config := serverConfigData(serviceIP, cluster)
-	if init {
-		config = initConfigData(cluster)
-	}
-	return &v1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Secret",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: util.ClusterNamespace(cluster),
-		},
-		Data: map[string][]byte{
-			"config.yaml": []byte(config),
-		},
-	}, nil
-}
-
-func serverConfigData(serviceIP string, cluster *v1alpha1.Cluster) string {
-	return "cluster-init: true\nserver: https://" + serviceIP + ":6443\n" + serverOptions(cluster)
-}
-
-func initConfigData(cluster *v1alpha1.Cluster) string {
-	return "cluster-init: true\n" + serverOptions(cluster)
-}
-
-func serverOptions(cluster *v1alpha1.Cluster) string {
-	var opts string
-
-	// TODO: generate token if not found
-	if cluster.Spec.Token != "" {
-		opts = "token: " + cluster.Spec.Token + "\n"
-	}
-	if cluster.Status.ClusterCIDR != "" {
-		opts = opts + "cluster-cidr: " + cluster.Status.ClusterCIDR + "\n"
-	}
-	if cluster.Status.ServiceCIDR != "" {
-		opts = opts + "service-cidr: " + cluster.Status.ServiceCIDR + "\n"
-	}
-	if cluster.Spec.ClusterDNS != "" {
-		opts = opts + "cluster-dns: " + cluster.Spec.ClusterDNS + "\n"
-	}
-	if len(cluster.Status.TLSSANs) > 0 {
-		opts = opts + "tls-san:\n"
-		for _, addr := range cluster.Status.TLSSANs {
-			opts = opts + "- " + addr + "\n"
-		}
-	}
-	// TODO: Add extra args to the options
-
-	return opts
-}

pkg/controller/cluster/pod.go

@@ -0,0 +1,241 @@
+package cluster
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+
+	certutil "github.com/rancher/dynamiclistener/cert"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	k3kcontroller "github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/certs"
+	"github.com/rancher/k3k/pkg/controller/cluster/server"
+	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	"github.com/rancher/k3k/pkg/log"
+	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.uber.org/zap"
+	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+	podController = "k3k-pod-controller"
+)
+
+type PodReconciler struct {
+	Client ctrlruntimeclient.Client
+	Scheme *runtime.Scheme
+	logger *log.Logger
+}
+
+// Add adds a new controller to the manager
+func AddPodController(ctx context.Context, mgr manager.Manager, logger *log.Logger) error {
+	// initialize a new Reconciler
+	reconciler := PodReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+		logger: logger.Named(podController),
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		Watches(&v1.Pod{}, handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &apps.StatefulSet{}, handler.OnlyControllerOwner())).
+		Named(podController).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: maxConcurrentReconciles,
+		}).
+		Complete(&reconciler)
+}
+
+func (p *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	log := p.logger.With("Pod", req.NamespacedName)
+
+	s := strings.Split(req.Name, "-")
+	if len(s) < 1 {
+		return reconcile.Result{}, nil
+	}
+	if s[0] != "k3k" {
+		return reconcile.Result{}, nil
+	}
+	clusterName := s[1]
+	var cluster v1alpha1.Cluster
+	if err := p.Client.Get(ctx, types.NamespacedName{Name: clusterName}, &cluster); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return reconcile.Result{}, err
+		}
+	}
+	matchingLabels := ctrlruntimeclient.MatchingLabels(map[string]string{"role": "server"})
+	listOpts := &ctrlruntimeclient.ListOptions{Namespace: req.Namespace}
+	matchingLabels.ApplyToList(listOpts)
+
+	var podList v1.PodList
+	if err := p.Client.List(ctx, &podList, listOpts); err != nil {
+		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
+	}
+	for _, pod := range podList.Items {
+		log.Info("Handle etcd server pod")
+		if err := p.handleServerPod(ctx, cluster, &pod, log); err != nil {
+			return reconcile.Result{}, err
+		}
+	}
+	return reconcile.Result{}, nil
+}
+
+func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cluster, pod *v1.Pod, log *zap.SugaredLogger) error {
+	if _, ok := pod.Labels["role"]; ok {
+		if pod.Labels["role"] != "server" {
+			return nil
+		}
+	} else {
+		return fmt.Errorf("server pod has no role label")
+	}
+	// if etcd pod is marked for deletion then we need to remove it from the etcd member list before deletion
+	if !pod.DeletionTimestamp.IsZero() {
+		// check if cluster is deleted then remove the finalizer from the pod
+		if cluster.Name == "" {
+			if controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) {
+				controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName)
+				if err := p.Client.Update(ctx, pod); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+		tlsConfig, err := p.getETCDTLS(ctx, &cluster, log)
+		if err != nil {
+			return err
+		}
+		// remove server from etcd
+		client, err := clientv3.New(clientv3.Config{
+			Endpoints: []string{
+				fmt.Sprintf("https://%s.%s:2379", server.ServiceName(cluster.Name), pod.Namespace),
+			},
+			TLS: tlsConfig,
+		})
+		if err != nil {
+			return err
+		}
+
+		if err := removePeer(ctx, client, pod.Name, pod.Status.PodIP, log); err != nil {
+			return err
+		}
+		// remove our finalizer from the list and update it.
+		if controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) {
+			controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName)
+			if err := p.Client.Update(ctx, pod); err != nil {
+				return err
+			}
+		}
+	}
+	if !controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) {
+		controllerutil.AddFinalizer(pod, etcdPodFinalizerName)
+		return p.Client.Update(ctx, pod)
+	}
+
+	return nil
+}
+
+func (p *PodReconciler) getETCDTLS(ctx context.Context, cluster *v1alpha1.Cluster, log *zap.SugaredLogger) (*tls.Config, error) {
+	log.Infow("generating etcd TLS client certificate", "Cluster", cluster.Name, "Namespace", cluster.Namespace)
+	token, err := p.clusterToken(ctx, cluster)
+	if err != nil {
+		return nil, err
+	}
+	endpoint := server.ServiceName(cluster.Name) + "." + cluster.Namespace
+	var b *bootstrap.ControlRuntimeBootstrap
+	if err := retry.OnError(k3kcontroller.Backoff, func(err error) bool {
+		return true
+	}, func() error {
+		var err error
+		b, err = bootstrap.DecodedBootstrap(token, endpoint)
+		return err
+	}); err != nil {
+		return nil, err
+	}
+
+	etcdCert, etcdKey, err := certs.CreateClientCertKey("etcd-client", nil, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, 0, b.ETCDServerCA.Content, b.ETCDServerCAKey.Content)
+	if err != nil {
+		return nil, err
+	}
+	clientCert, err := tls.X509KeyPair(etcdCert, etcdKey)
+	if err != nil {
+		return nil, err
+	}
+	// create rootCA CertPool
+	cert, err := certutil.ParseCertsPEM([]byte(b.ETCDServerCA.Content))
+	if err != nil {
+		return nil, err
+	}
+	pool := x509.NewCertPool()
+	pool.AddCert(cert[0])
+
+	return &tls.Config{
+		RootCAs:      pool,
+		Certificates: []tls.Certificate{clientCert},
+	}, nil
+}
+
+// removePeer removes a peer from the cluster. The peer name and IP address must both match.
+func removePeer(ctx context.Context, client *clientv3.Client, name, address string, log *zap.SugaredLogger) error {
+	ctx, cancel := context.WithTimeout(ctx, memberRemovalTimeout)
+	defer cancel()
+	members, err := client.MemberList(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, member := range members.Members {
+		if !strings.Contains(member.Name, name) {
+			continue
+		}
+		for _, peerURL := range member.PeerURLs {
+			u, err := url.Parse(peerURL)
+			if err != nil {
+				return err
+			}
+			if u.Hostname() == address {
+				log.Infow("Removing member from etcd", "name", member.Name, "id", member.ID, "address", address)
+				_, err := client.MemberRemove(ctx, member.ID)
+				if errors.Is(err, rpctypes.ErrGRPCMemberNotFound) {
+					return nil
+				}
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (p *PodReconciler) clusterToken(ctx context.Context, cluster *v1alpha1.Cluster) (string, error) {
+	var tokenSecret v1.Secret
+	nn := types.NamespacedName{
+		Name:      TokenSecretName(cluster.Name),
+		Namespace: cluster.Namespace,
+	}
+	if cluster.Spec.TokenSecretRef != nil {
+		nn.Name = TokenSecretName(cluster.Name)
+	}
+	if err := p.Client.Get(ctx, nn, &tokenSecret); err != nil {
+		return "", err
+	}
+	if _, ok := tokenSecret.Data["token"]; !ok {
+		return "", fmt.Errorf("no token field in secret %s/%s", nn.Namespace, nn.Name)
+	}
+	return string(tokenSecret.Data["token"]), nil
+}

pkg/controller/cluster/server/bootstrap/bootstrap.go

@@ -0,0 +1,173 @@
+package bootstrap
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
+)
+
+type ControlRuntimeBootstrap struct {
+	ServerCA        content `json:"serverCA"`
+	ServerCAKey     content `json:"serverCAKey"`
+	ClientCA        content `json:"clientCA"`
+	ClientCAKey     content `json:"clientCAKey"`
+	ETCDServerCA    content `json:"etcdServerCA"`
+	ETCDServerCAKey content `json:"etcdServerCAKey"`
+}
+
+type content struct {
+	Timestamp string
+	Content   string
+}
+
+// Generate generates the bootstrap for the cluster:
+// 1- use the server token to get the bootstrap data from k3s
+// 2- save the bootstrap data as a secret
+func Generate(ctx context.Context, cluster *v1alpha1.Cluster, ip, token string) (*v1.Secret, error) {
+	var bootstrap *ControlRuntimeBootstrap
+	if err := retry.OnError(controller.Backoff, func(err error) bool {
+		return true
+	}, func() error {
+		var err error
+		bootstrap, err = requestBootstrap(token, ip)
+		return err
+	}); err != nil {
+		return nil, err
+	}
+
+	if err := decodeBootstrap(bootstrap); err != nil {
+		return nil, err
+	}
+
+	bootstrapData, err := json.Marshal(bootstrap)
+	if err != nil {
+		return nil, err
+	}
+	return &v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
+			Namespace: cluster.Namespace,
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion: cluster.APIVersion,
+					Kind:       cluster.Kind,
+					Name:       cluster.Name,
+					UID:        cluster.UID,
+				},
+			},
+		},
+		Data: map[string][]byte{
+			"bootstrap": bootstrapData,
+		},
+	}, nil
+
+}
+
+func requestBootstrap(token, serverIP string) (*ControlRuntimeBootstrap, error) {
+	url := "https://" + serverIP + ":6443/v1-k3s/server-bootstrap"
+
+	client := http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+		Timeout: 5 * time.Second,
+	}
+
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Authorization", "Basic "+basicAuth("server", token))
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var runtimeBootstrap ControlRuntimeBootstrap
+	if err := json.NewDecoder(resp.Body).Decode(&runtimeBootstrap); err != nil {
+		return nil, err
+	}
+
+	return &runtimeBootstrap, nil
+}
+
+func basicAuth(username, password string) string {
+	auth := username + ":" + password
+	return base64.StdEncoding.EncodeToString([]byte(auth))
+}
+
+func decodeBootstrap(bootstrap *ControlRuntimeBootstrap) error {
+	//client-ca
+	decoded, err := base64.StdEncoding.DecodeString(bootstrap.ClientCA.Content)
+	if err != nil {
+		return err
+	}
+	bootstrap.ClientCA.Content = string(decoded)
+
+	//client-ca-key
+	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ClientCAKey.Content)
+	if err != nil {
+		return err
+	}
+	bootstrap.ClientCAKey.Content = string(decoded)
+
+	//server-ca
+	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ServerCA.Content)
+	if err != nil {
+		return err
+	}
+	bootstrap.ServerCA.Content = string(decoded)
+
+	//server-ca-key
+	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ServerCAKey.Content)
+	if err != nil {
+		return err
+	}
+	bootstrap.ServerCAKey.Content = string(decoded)
+
+	//etcd-ca
+	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ETCDServerCA.Content)
+	if err != nil {
+		return err
+	}
+	bootstrap.ETCDServerCA.Content = string(decoded)
+
+	//etcd-ca-key
+	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ETCDServerCAKey.Content)
+	if err != nil {
+		return err
+	}
+	bootstrap.ETCDServerCAKey.Content = string(decoded)
+
+	return nil
+}
+
+func DecodedBootstrap(token, ip string) (*ControlRuntimeBootstrap, error) {
+	bootstrap, err := requestBootstrap(token, ip)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := decodeBootstrap(bootstrap); err != nil {
+		return nil, err
+	}
+
+	return bootstrap, nil
+}

pkg/controller/cluster/server/config.go

@@ -0,0 +1,83 @@
+package server
+
+import (
+	"fmt"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/cluster/agent"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (s *Server) Config(init bool, serviceIP string) (*v1.Secret, error) {
+	name := configSecretName(s.cluster.Name, init)
+	s.cluster.Status.TLSSANs = append(s.cluster.Spec.TLSSANs,
+		serviceIP,
+		ServiceName(s.cluster.Name),
+		fmt.Sprintf("%s.%s", ServiceName(s.cluster.Name), s.cluster.Namespace),
+	)
+
+	config := serverConfigData(serviceIP, s.cluster, s.token)
+	if init {
+		config = initConfigData(s.cluster, s.token)
+	}
+	return &v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: s.cluster.Namespace,
+		},
+		Data: map[string][]byte{
+			"config.yaml": []byte(config),
+		},
+	}, nil
+}
+
+func serverConfigData(serviceIP string, cluster *v1alpha1.Cluster, token string) string {
+	return "cluster-init: true\nserver: https://" + serviceIP + ":6443\n" + serverOptions(cluster, token)
+}
+
+func initConfigData(cluster *v1alpha1.Cluster, token string) string {
+	return "cluster-init: true\n" + serverOptions(cluster, token)
+}
+
+func serverOptions(cluster *v1alpha1.Cluster, token string) string {
+	var opts string
+
+	// TODO: generate token if not found
+	if token != "" {
+		opts = "token: " + token + "\n"
+	}
+	if cluster.Status.ClusterCIDR != "" {
+		opts = opts + "cluster-cidr: " + cluster.Status.ClusterCIDR + "\n"
+	}
+	if cluster.Status.ServiceCIDR != "" {
+		opts = opts + "service-cidr: " + cluster.Status.ServiceCIDR + "\n"
+	}
+	if cluster.Spec.ClusterDNS != "" {
+		opts = opts + "cluster-dns: " + cluster.Spec.ClusterDNS + "\n"
+	}
+	if len(cluster.Status.TLSSANs) > 0 {
+		opts = opts + "tls-san:\n"
+		for _, addr := range cluster.Status.TLSSANs {
+			opts = opts + "- " + addr + "\n"
+		}
+	}
+	if cluster.Spec.Mode != agent.VirtualNodeMode {
+		opts = opts + "disable-agent: true\negress-selector-mode: disabled\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage"
+	}
+	// TODO: Add extra args to the options
+
+	return opts
+}
+
+func configSecretName(clusterName string, init bool) string {
+	if !init {
+		return controller.SafeConcatNameWithPrefix(clusterName, configName)
+	}
+	return controller.SafeConcatNameWithPrefix(clusterName, initConfigName)
+}

pkg/controller/cluster/server/ingress.go

@@ -3,7 +3,7 @@ package server
 import (
 	"context"
 
-	"github.com/rancher/k3k/pkg/controller/util"
+	"github.com/rancher/k3k/pkg/controller"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -15,10 +15,13 @@ const (
 	nginxSSLPassthroughAnnotation  = "nginx.ingress.kubernetes.io/ssl-passthrough"
 	nginxBackendProtocolAnnotation = "nginx.ingress.kubernetes.io/backend-protocol"
 	nginxSSLRedirectAnnotation     = "nginx.ingress.kubernetes.io/ssl-redirect"
+
+	serverPort = 6443
+	etcdPort   = 2379
 )
 
 func (s *Server) Ingress(ctx context.Context, client client.Client) (*networkingv1.Ingress, error) {
-	addresses, err := util.Addresses(ctx, client)
+	addresses, err := controller.Addresses(ctx, client)
 	if err != nil {
 		return nil, err
 	}
@@ -29,8 +32,8 @@ func (s *Server) Ingress(ctx context.Context, client client.Client) (*networking
 			APIVersion: "networking.k8s.io/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      s.cluster.Name + "-server-ingress",
-			Namespace: util.ClusterNamespace(s.cluster),
+			Name:      controller.SafeConcatNameWithPrefix(s.cluster.Name, "ingress"),
+			Namespace: s.cluster.Namespace,
 		},
 		Spec: networkingv1.IngressSpec{
 			IngressClassName: &s.cluster.Spec.Expose.Ingress.IngressClassName,
@@ -57,9 +60,9 @@ func (s *Server) ingressRules(addresses []string) []networkingv1.IngressRule {
 							PathType: &pathTypePrefix,
 							Backend: networkingv1.IngressBackend{
 								Service: &networkingv1.IngressServiceBackend{
-									Name: "k3k-server-service",
+									Name: ServiceName(s.cluster.Name),
 									Port: networkingv1.ServiceBackendPort{
-										Number: port,
+										Number: serverPort,
 									},
 								},
 							},

pkg/controller/cluster/server/kubeconfig.go

@@ -1,265 +0,0 @@
-package server
-
-import (
-	"context"
-	"crypto"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/base64"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"time"
-
-	certutil "github.com/rancher/dynamiclistener/cert"
-	"github.com/rancher/k3k/pkg/controller/util"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apiserver/pkg/authentication/user"
-	"k8s.io/client-go/tools/clientcmd"
-	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
-	"k8s.io/client-go/util/retry"
-)
-
-const (
-	adminCommonName = "system:admin"
-	port            = 6443
-)
-
-type ControlRuntimeBootstrap struct {
-	ServerCA        content
-	ServerCAKey     content
-	ClientCA        content
-	ClientCAKey     content
-	ETCDServerCA    content
-	ETCDServerCAKey content
-}
-
-type content struct {
-	Timestamp string
-	Content   string
-}
-
-// GenerateNewKubeConfig generates the kubeconfig for the cluster:
-// 1- use the server token to get the bootstrap data from k3s
-// 2- generate client admin cert/key
-// 3- use the ca cert from the bootstrap data & admin cert/key to write a new kubeconfig
-// 4- save the new kubeconfig as a secret
-func (s *Server) GenerateNewKubeConfig(ctx context.Context, ip string) (*v1.Secret, error) {
-	token := s.cluster.Spec.Token
-
-	var bootstrap *ControlRuntimeBootstrap
-	if err := retry.OnError(retry.DefaultBackoff, func(err error) bool {
-		return true
-	}, func() error {
-		var err error
-		bootstrap, err = requestBootstrap(token, ip)
-		return err
-	}); err != nil {
-		return nil, err
-	}
-
-	if err := decodeBootstrap(bootstrap); err != nil {
-		return nil, err
-	}
-
-	adminCert, adminKey, err := CreateClientCertKey(
-		adminCommonName, []string{user.SystemPrivilegedGroup},
-		nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
-		bootstrap.ClientCA.Content,
-		bootstrap.ClientCAKey.Content)
-	if err != nil {
-		return nil, err
-	}
-
-	url := fmt.Sprintf("https://%s:%d", ip, port)
-	kubeconfigData, err := kubeconfig(url, []byte(bootstrap.ServerCA.Content), adminCert, adminKey)
-	if err != nil {
-		return nil, err
-	}
-
-	return &v1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Secret",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      s.cluster.Name + "-kubeconfig",
-			Namespace: util.ClusterNamespace(s.cluster),
-		},
-		Data: map[string][]byte{
-			"kubeconfig.yaml": kubeconfigData,
-		},
-	}, nil
-
-}
-
-func requestBootstrap(token, serverIP string) (*ControlRuntimeBootstrap, error) {
-	url := "https://" + serverIP + ":6443/v1-k3s/server-bootstrap"
-
-	client := http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
-		},
-		Timeout: 5 * time.Second,
-	}
-
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Add("Authorization", "Basic "+basicAuth("server", token))
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	var runtimeBootstrap ControlRuntimeBootstrap
-	if err := json.NewDecoder(resp.Body).Decode(&runtimeBootstrap); err != nil {
-		return nil, err
-	}
-
-	return &runtimeBootstrap, nil
-}
-
-func CreateClientCertKey(commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCert, caKey string) ([]byte, []byte, error) {
-	caKeyPEM, err := certutil.ParsePrivateKeyPEM([]byte(caKey))
-	if err != nil {
-		return nil, nil, err
-	}
-
-	caCertPEM, err := certutil.ParseCertsPEM([]byte(caCert))
-	if err != nil {
-		return nil, nil, err
-	}
-
-	b, err := generateKey()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	key, err := certutil.ParsePrivateKeyPEM(b)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	cfg := certutil.Config{
-		CommonName:   commonName,
-		Organization: organization,
-		Usages:       extKeyUsage,
-	}
-	if altNames != nil {
-		cfg.AltNames = *altNames
-	}
-	cert, err := certutil.NewSignedCert(cfg, key.(crypto.Signer), caCertPEM[0], caKeyPEM.(crypto.Signer))
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCertPEM[0])...), b, nil
-}
-
-func generateKey() (data []byte, err error) {
-	generatedData, err := certutil.MakeEllipticPrivateKeyPEM()
-	if err != nil {
-		return nil, fmt.Errorf("error generating key: %v", err)
-	}
-
-	return generatedData, nil
-}
-
-func kubeconfig(url string, serverCA, clientCert, clientKey []byte) ([]byte, error) {
-	config := clientcmdapi.NewConfig()
-
-	cluster := clientcmdapi.NewCluster()
-	cluster.CertificateAuthorityData = serverCA
-	cluster.Server = url
-
-	authInfo := clientcmdapi.NewAuthInfo()
-	authInfo.ClientCertificateData = clientCert
-	authInfo.ClientKeyData = clientKey
-
-	context := clientcmdapi.NewContext()
-	context.AuthInfo = "default"
-	context.Cluster = "default"
-
-	config.Clusters["default"] = cluster
-	config.AuthInfos["default"] = authInfo
-	config.Contexts["default"] = context
-	config.CurrentContext = "default"
-
-	kubeconfig, err := clientcmd.Write(*config)
-	if err != nil {
-		return nil, err
-	}
-
-	return kubeconfig, nil
-}
-
-func basicAuth(username, password string) string {
-	auth := username + ":" + password
-	return base64.StdEncoding.EncodeToString([]byte(auth))
-}
-
-func decodeBootstrap(bootstrap *ControlRuntimeBootstrap) error {
-	//client-ca
-	decoded, err := base64.StdEncoding.DecodeString(bootstrap.ClientCA.Content)
-	if err != nil {
-		return err
-	}
-	bootstrap.ClientCA.Content = string(decoded)
-
-	//client-ca-key
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ClientCAKey.Content)
-	if err != nil {
-		return err
-	}
-	bootstrap.ClientCAKey.Content = string(decoded)
-
-	//server-ca
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ServerCA.Content)
-	if err != nil {
-		return err
-	}
-	bootstrap.ServerCA.Content = string(decoded)
-
-	//server-ca-key
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ServerCAKey.Content)
-	if err != nil {
-		return err
-	}
-	bootstrap.ServerCAKey.Content = string(decoded)
-
-	//etcd-ca
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ETCDServerCA.Content)
-	if err != nil {
-		return err
-	}
-	bootstrap.ETCDServerCA.Content = string(decoded)
-
-	//etcd-ca-key
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ETCDServerCAKey.Content)
-	if err != nil {
-		return err
-	}
-	bootstrap.ETCDServerCAKey.Content = string(decoded)
-
-	return nil
-}
-
-func DecodedBootstrap(token, ip string) (*ControlRuntimeBootstrap, error) {
-	bootstrap, err := requestBootstrap(token, ip)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := decodeBootstrap(bootstrap); err != nil {
-		return nil, err
-	}
-
-	return bootstrap, nil
-}

pkg/controller/cluster/server/server.go

@@ -5,24 +5,25 @@ import (
 	"strings"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	"github.com/rancher/k3k/pkg/controller/util"
+	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/cluster/agent"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
-	serverName         = "k3k-"
-	k3kSystemNamespace = serverName + "system"
-	initServerName     = serverName + "init-server"
-	initContainerName  = serverName + "server-check"
-	initContainerImage = "alpine/curl"
+	k3kSystemNamespace = "k3k-system"
+	serverName         = "server"
+	configName         = "server-config"
+	initConfigName     = "init-server-config"
 
+	ServerPort        = 6443
 	EphermalNodesType = "ephermal"
 	DynamicNodesType  = "dynamic"
 )
@@ -31,23 +32,43 @@ const (
 type Server struct {
 	cluster *v1alpha1.Cluster
 	client  client.Client
+	mode    string
+	token   string
 }
 
-func New(cluster *v1alpha1.Cluster, client client.Client) *Server {
+func New(cluster *v1alpha1.Cluster, client client.Client, token, mode string) *Server {
 	return &Server{
 		cluster: cluster,
 		client:  client,
+		token:   token,
+		mode:    mode,
 	}
 }
 
-func (s *Server) podSpec(ctx context.Context, image, name string, persistent bool) v1.PodSpec {
+func (s *Server) podSpec(image, name string, persistent bool, affinitySelector *metav1.LabelSelector) v1.PodSpec {
+	var limit v1.ResourceList
+	if s.cluster.Spec.Limit != nil && s.cluster.Spec.Limit.ServerLimit != nil {
+		limit = s.cluster.Spec.Limit.ServerLimit
+	}
 	podSpec := v1.PodSpec{
+		NodeSelector:      s.cluster.Spec.NodeSelector,
+		PriorityClassName: s.cluster.Spec.PriorityClass,
+		Affinity: &v1.Affinity{
+			PodAntiAffinity: &v1.PodAntiAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
+					{
+						LabelSelector: affinitySelector,
+						TopologyKey:   "kubernetes.io/hostname",
+					},
+				},
+			},
+		},
 		Volumes: []v1.Volume{
 			{
 				Name: "initconfig",
 				VolumeSource: v1.VolumeSource{
 					Secret: &v1.SecretVolumeSource{
-						SecretName: "k3k-init-server-config",
+						SecretName: configSecretName(s.cluster.Name, true),
 						Items: []v1.KeyToPath{
 							{
 								Key:  "config.yaml",
@@ -61,7 +82,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 				Name: "config",
 				VolumeSource: v1.VolumeSource{
 					Secret: &v1.SecretVolumeSource{
-						SecretName: "k3k-server-config",
+						SecretName: configSecretName(s.cluster.Name, false),
 						Items: []v1.KeyToPath{
 							{
 								Key:  "config.yaml",
@@ -100,6 +121,9 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 			{
 				Name:  name,
 				Image: image,
+				Resources: v1.ResourceRequirements{
+					Limits: limit,
+				},
 				Env: []v1.EnvVar{
 					{
 						Name: "POD_NAME",
@@ -110,15 +134,14 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 						},
 					},
 				},
-				SecurityContext: &v1.SecurityContext{
-					Privileged: pointer.Bool(true),
-				},
 				Command: []string{
 					"/bin/sh",
 					"-c",
-					`if [ ${POD_NAME: -1} == 0 ]; then 
-				       /bin/k3s server --config /opt/rancher/k3s/init/config.yaml ` + strings.Join(s.cluster.Spec.ServerArgs, " ") + `
-					else /bin/k3s server --config /opt/rancher/k3s/server/config.yaml ` + strings.Join(s.cluster.Spec.ServerArgs, " ") + `
+					`
+					if [ ${POD_NAME: -1} == 0 ]; then 
+						/bin/k3s server --config /opt/rancher/k3s/init/config.yaml ` + strings.Join(s.cluster.Spec.ServerArgs, " ") + `
+					else 
+						/bin/k3s server --config /opt/rancher/k3s/server/config.yaml ` + strings.Join(s.cluster.Spec.ServerArgs, " ") + `
 					fi   
 					`,
 				},
@@ -185,7 +208,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 		)
 	}
 
-	// Adding readiness probes to deployment
+	// Adding readiness probes to statefulset
 	podSpec.Containers[0].ReadinessProbe = &v1.Probe{
 		InitialDelaySeconds: 60,
 		FailureThreshold:    5,
@@ -193,26 +216,30 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 		ProbeHandler: v1.ProbeHandler{
 			TCPSocket: &v1.TCPSocketAction{
 				Port: intstr.FromInt(6443),
-				Host: "127.0.0.1",
 			},
 		},
 	}
-
+	// start the pod unprivileged in shared mode
+	if s.mode == agent.VirtualNodeMode {
+		podSpec.Containers[0].SecurityContext = &v1.SecurityContext{
+			Privileged: ptr.To(true),
+		}
+	}
 	return podSpec
 }
 
-func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster) (*apps.StatefulSet, error) {
+func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error) {
 	var (
 		replicas   int32
 		pvClaims   []v1.PersistentVolumeClaim
 		persistent bool
 	)
-	image := util.K3SImage(cluster)
-	name := serverName + "server"
+	image := controller.K3SImage(s.cluster)
+	name := controller.SafeConcatNameWithPrefix(s.cluster.Name, serverName)
 
-	replicas = *cluster.Spec.Servers
+	replicas = *s.cluster.Spec.Servers
 
-	if cluster.Spec.Persistence != nil && cluster.Spec.Persistence.Type != EphermalNodesType {
+	if s.cluster.Spec.Persistence != nil && s.cluster.Spec.Persistence.Type != EphermalNodesType {
 		persistent = true
 		pvClaims = []v1.PersistentVolumeClaim{
 			{
@@ -222,14 +249,14 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster)
 				},
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "varlibrancherk3s",
-					Namespace: util.ClusterNamespace(cluster),
+					Namespace: s.cluster.Namespace,
 				},
 				Spec: v1.PersistentVolumeClaimSpec{
 					AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
-					StorageClassName: &cluster.Spec.Persistence.StorageClassName,
-					Resources: v1.ResourceRequirements{
+					StorageClassName: &s.cluster.Spec.Persistence.StorageClassName,
+					Resources: v1.VolumeResourceRequirements{
 						Requests: v1.ResourceList{
-							"storage": resource.MustParse(cluster.Spec.Persistence.StorageRequestSize),
+							"storage": resource.MustParse(s.cluster.Spec.Persistence.StorageRequestSize),
 						},
 					},
 				},
@@ -241,16 +268,16 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster)
 				},
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "varlibkubelet",
-					Namespace: util.ClusterNamespace(cluster),
+					Namespace: s.cluster.Namespace,
 				},
 				Spec: v1.PersistentVolumeClaimSpec{
-					Resources: v1.ResourceRequirements{
+					Resources: v1.VolumeResourceRequirements{
 						Requests: v1.ResourceList{
-							"storage": resource.MustParse(cluster.Spec.Persistence.StorageRequestSize),
+							"storage": resource.MustParse(s.cluster.Spec.Persistence.StorageRequestSize),
 						},
 					},
 					AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
-					StorageClassName: &cluster.Spec.Persistence.StorageClassName,
+					StorageClassName: &s.cluster.Spec.Persistence.StorageClassName,
 				},
 			},
 		}
@@ -258,7 +285,6 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster)
 
 	var volumes []v1.Volume
 	var volumeMounts []v1.VolumeMount
-
 	for _, addon := range s.cluster.Spec.Addons {
 		namespace := k3kSystemNamespace
 		if addon.SecretNamespace != "" {
@@ -282,7 +308,7 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster)
 			},
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      addons.Name,
-				Namespace: util.ClusterNamespace(s.cluster),
+				Namespace: s.cluster.Namespace,
 			},
 			Data: make(map[string][]byte, len(addons.Data)),
 		}
@@ -314,7 +340,14 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster)
 		volumeMounts = append(volumeMounts, volumeMount)
 	}
 
-	podSpec := s.podSpec(ctx, image, name, persistent)
+	selector := metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"cluster": s.cluster.Name,
+			"role":    "server",
+		},
+	}
+
+	podSpec := s.podSpec(image, name, persistent, &selector)
 	podSpec.Volumes = append(podSpec.Volumes, volumes...)
 	podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...)
 
@@ -324,25 +357,18 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster)
 			APIVersion: "apps/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      cluster.Name + "-" + name,
-			Namespace: util.ClusterNamespace(cluster),
+			Name:      name,
+			Namespace: s.cluster.Namespace,
+			Labels:    selector.MatchLabels,
 		},
 		Spec: apps.StatefulSetSpec{
-			Replicas:    &replicas,
-			ServiceName: cluster.Name + "-" + name + "-headless",
-			Selector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{
-					"cluster": cluster.Name,
-					"role":    "server",
-				},
-			},
+			Replicas:             &replicas,
+			ServiceName:          headlessServiceName(s.cluster.Name),
+			Selector:             &selector,
 			VolumeClaimTemplates: pvClaims,
 			Template: v1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						"cluster": cluster.Name,
-						"role":    "server",
-					},
+					Labels: selector.MatchLabels,
 				},
 				Spec: podSpec,
 			},

pkg/controller/cluster/server/service.go

@@ -2,7 +2,7 @@ package server
 
 import (
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	"github.com/rancher/k3k/pkg/controller/util"
+	"github.com/rancher/k3k/pkg/controller"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -23,8 +23,8 @@ func (s *Server) Service(cluster *v1alpha1.Cluster) *v1.Service {
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "k3k-server-service",
-			Namespace: util.ClusterNamespace(cluster),
+			Name:      ServiceName(s.cluster.Name),
+			Namespace: cluster.Namespace,
 		},
 		Spec: v1.ServiceSpec{
 			Type: serviceType,
@@ -36,48 +36,55 @@ func (s *Server) Service(cluster *v1alpha1.Cluster) *v1.Service {
 				{
 					Name:     "k3s-server-port",
 					Protocol: v1.ProtocolTCP,
-					Port:     port,
+					Port:     serverPort,
 				},
 				{
 					Name:     "k3s-etcd-port",
 					Protocol: v1.ProtocolTCP,
-					Port:     2379,
+					Port:     etcdPort,
 				},
 			},
 		},
 	}
 }
 
-func (s *Server) StatefulServerService(cluster *v1alpha1.Cluster) *v1.Service {
-	name := serverName
+func (s *Server) StatefulServerService() *v1.Service {
 	return &v1.Service{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Service",
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      cluster.Name + "-" + name + "-headless",
-			Namespace: util.ClusterNamespace(cluster),
+			Name:      headlessServiceName(s.cluster.Name),
+			Namespace: s.cluster.Namespace,
 		},
 		Spec: v1.ServiceSpec{
 			Type:      v1.ServiceTypeClusterIP,
 			ClusterIP: v1.ClusterIPNone,
 			Selector: map[string]string{
-				"cluster": cluster.Name,
+				"cluster": s.cluster.Name,
 				"role":    "server",
 			},
 			Ports: []v1.ServicePort{
 				{
 					Name:     "k3s-server-port",
 					Protocol: v1.ProtocolTCP,
-					Port:     6443,
+					Port:     serverPort,
 				},
 				{
 					Name:     "k3s-etcd-port",
 					Protocol: v1.ProtocolTCP,
-					Port:     2379,
+					Port:     etcdPort,
 				},
 			},
 		},
 	}
 }
+
+func ServiceName(clusterName string) string {
+	return controller.SafeConcatNameWithPrefix(clusterName, "service")
+}
+
+func headlessServiceName(clusterName string) string {
+	return controller.SafeConcatNameWithPrefix(clusterName, "service", "headless")
+}

pkg/controller/cluster/token.go

@@ -0,0 +1,98 @@
+package cluster
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (c *ClusterReconciler) token(ctx context.Context, cluster *v1alpha1.Cluster) (string, error) {
+	if cluster.Spec.TokenSecretRef == nil {
+		return c.ensureTokenSecret(ctx, cluster)
+	}
+	// get token data from secretRef
+	nn := types.NamespacedName{
+		Name:      cluster.Spec.TokenSecretRef.Name,
+		Namespace: cluster.Spec.TokenSecretRef.Namespace,
+	}
+	var tokenSecret v1.Secret
+	if err := c.Client.Get(ctx, nn, &tokenSecret); err != nil {
+		return "", err
+	}
+	if _, ok := tokenSecret.Data["token"]; !ok {
+		return "", fmt.Errorf("no token field in secret %s/%s", nn.Namespace, nn.Name)
+	}
+	return string(tokenSecret.Data["token"]), nil
+}
+
+func (c *ClusterReconciler) ensureTokenSecret(ctx context.Context, cluster *v1alpha1.Cluster) (string, error) {
+	// check if the secret is already created
+	var (
+		tokenSecret v1.Secret
+		nn          = types.NamespacedName{
+			Name:      TokenSecretName(cluster.Name),
+			Namespace: cluster.Namespace,
+		}
+	)
+	if err := c.Client.Get(ctx, nn, &tokenSecret); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return "", err
+		}
+	}
+
+	if tokenSecret.Data != nil {
+		return string(tokenSecret.Data["token"]), nil
+	}
+	c.logger.Info("Token secret is not specified, creating a random token")
+	token, err := random(16)
+	if err != nil {
+		return "", err
+	}
+	tokenSecret = TokenSecretObj(token, cluster.Name, cluster.Namespace)
+	if err := controllerutil.SetControllerReference(cluster, &tokenSecret, c.Scheme); err != nil {
+		return "", err
+	}
+	if err := c.ensure(ctx, &tokenSecret, false); err != nil {
+		return "", err
+	}
+	return token, nil
+
+}
+
+func random(size int) (string, error) {
+	token := make([]byte, size)
+	_, err := rand.Read(token)
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(token), err
+}
+
+func TokenSecretObj(token, name, namespace string) v1.Secret {
+	return v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      TokenSecretName(name),
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			"token": []byte(token),
+		},
+	}
+}
+
+func TokenSecretName(clusterName string) string {
+	return controller.SafeConcatNameWithPrefix(clusterName, "token")
+}

pkg/controller/clusterset/clusterset.go

@@ -0,0 +1,332 @@
+package clusterset
+
+import (
+	"context"
+	"errors"
+	"reflect"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	k3kcontroller "github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/log"
+	"go.uber.org/zap"
+	v1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+	clusterSetController    = "k3k-clusterset-controller"
+	allTrafficCIDR          = "0.0.0.0/0"
+	maxConcurrentReconciles = 1
+)
+
+type ClusterSetReconciler struct {
+	Client      ctrlruntimeclient.Client
+	Scheme      *runtime.Scheme
+	ClusterCIDR string
+	logger      *log.Logger
+}
+
+// Add adds a new controller to the manager
+func Add(ctx context.Context, mgr manager.Manager, clusterCIDR string, logger *log.Logger) error {
+	// initialize a new Reconciler
+	reconciler := ClusterSetReconciler{
+		Client:      mgr.GetClient(),
+		Scheme:      mgr.GetScheme(),
+		ClusterCIDR: clusterCIDR,
+		logger:      logger.Named(clusterSetController),
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.ClusterSet{}).
+		Owns(&networkingv1.NetworkPolicy{}).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: maxConcurrentReconciles,
+		}).
+		Watches(
+			&v1.Namespace{},
+			handler.EnqueueRequestsFromMapFunc(namespaceEventHandler(reconciler)),
+			builder.WithPredicates(namespaceLabelsPredicate()),
+		).
+		Watches(
+			&v1alpha1.Cluster{},
+			handler.EnqueueRequestsFromMapFunc(sameNamespaceEventHandler(reconciler)),
+		).
+		Complete(&reconciler)
+}
+
+// namespaceEventHandler will enqueue reconciling requests for all the ClusterSets in the changed namespace
+func namespaceEventHandler(reconciler ClusterSetReconciler) handler.MapFunc {
+	return func(ctx context.Context, obj client.Object) []reconcile.Request {
+		var requests []reconcile.Request
+		var set v1alpha1.ClusterSetList
+
+		_ = reconciler.Client.List(ctx, &set, client.InNamespace(obj.GetName()))
+		for _, clusterSet := range set.Items {
+			requests = append(requests, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Name:      clusterSet.Name,
+					Namespace: obj.GetName(),
+				},
+			})
+		}
+
+		return requests
+	}
+}
+
+// sameNamespaceEventHandler will enqueue reconciling requests for all the ClusterSets in the changed namespace
+func sameNamespaceEventHandler(reconciler ClusterSetReconciler) handler.MapFunc {
+	return func(ctx context.Context, obj client.Object) []reconcile.Request {
+		var requests []reconcile.Request
+		var set v1alpha1.ClusterSetList
+
+		_ = reconciler.Client.List(ctx, &set, client.InNamespace(obj.GetNamespace()))
+		for _, clusterSet := range set.Items {
+			requests = append(requests, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Name:      clusterSet.Name,
+					Namespace: obj.GetNamespace(),
+				},
+			})
+		}
+
+		return requests
+	}
+}
+
+// namespaceLabelsPredicate returns a predicate that will allow a reconciliation if the labels of a Namespace changed
+func namespaceLabelsPredicate() predicate.Predicate {
+	return predicate.Funcs{
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			oldObj := e.ObjectOld.(*v1.Namespace)
+			newObj := e.ObjectNew.(*v1.Namespace)
+
+			return !reflect.DeepEqual(oldObj.Labels, newObj.Labels)
+		},
+	}
+}
+
+func (c *ClusterSetReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	log := c.logger.With("ClusterSet", req.NamespacedName)
+
+	var clusterSet v1alpha1.ClusterSet
+	if err := c.Client.Get(ctx, req.NamespacedName, &clusterSet); err != nil {
+		return reconcile.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if err := c.reconcileNetworkPolicy(ctx, log, &clusterSet); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	if err := c.reconcileNamespacePodSecurityLabels(ctx, log, &clusterSet); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	if err := c.reconcileClusters(ctx, log, &clusterSet); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	// TODO: Add resource quota for clustersets
+	// if clusterSet.Spec.MaxLimits != nil {
+	// 	quota := v1.ResourceQuota{
+	// 		ObjectMeta: metav1.ObjectMeta{
+	// 			Name:      "clusterset-quota",
+	// 			Namespace: clusterSet.Namespace,
+	// 			OwnerReferences: []metav1.OwnerReference{
+	// 				{
+	// 					UID:        clusterSet.UID,
+	// 					Name:       clusterSet.Name,
+	// 					APIVersion: clusterSet.APIVersion,
+	// 					Kind:       clusterSet.Kind,
+	// 				},
+	// 			},
+	// 		},
+	// 	}
+	// 	quota.Spec.Hard = clusterSet.Spec.MaxLimits
+	// 	if err := c.Client.Create(ctx, &quota); err != nil {
+	// 		return reconcile.Result{}, fmt.Errorf("unable to create resource quota from cluster set: %w", err)
+	// 	}
+	// }
+	return reconcile.Result{}, nil
+}
+
+func (c *ClusterSetReconciler) reconcileNetworkPolicy(ctx context.Context, log *zap.SugaredLogger, clusterSet *v1alpha1.ClusterSet) error {
+	log.Info("reconciling NetworkPolicy")
+
+	networkPolicy, err := netpol(ctx, c.ClusterCIDR, clusterSet, c.Client)
+	if err != nil {
+		return err
+	}
+
+	if err = ctrl.SetControllerReference(clusterSet, networkPolicy, c.Scheme); err != nil {
+		return err
+	}
+
+	// if disabled then delete the existing network policy
+	if clusterSet.Spec.DisableNetworkPolicy {
+		err := c.Client.Delete(ctx, networkPolicy)
+		return client.IgnoreNotFound(err)
+	}
+
+	// otherwise try to create/update
+	err = c.Client.Create(ctx, networkPolicy)
+	if apierrors.IsAlreadyExists(err) {
+		return c.Client.Update(ctx, networkPolicy)
+	}
+
+	return err
+}
+
+func netpol(ctx context.Context, clusterCIDR string, clusterSet *v1alpha1.ClusterSet, client client.Client) (*networkingv1.NetworkPolicy, error) {
+	var cidrList []string
+
+	if clusterCIDR != "" {
+		cidrList = []string{clusterCIDR}
+	} else {
+		var nodeList v1.NodeList
+		if err := client.List(ctx, &nodeList); err != nil {
+			return nil, err
+		}
+		for _, node := range nodeList.Items {
+			cidrList = append(cidrList, node.Spec.PodCIDRs...)
+		}
+	}
+
+	return &networkingv1.NetworkPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+			Namespace: clusterSet.Namespace,
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "NetworkPolicy",
+			APIVersion: "networking.k8s.io/v1",
+		},
+		Spec: networkingv1.NetworkPolicySpec{
+			PolicyTypes: []networkingv1.PolicyType{
+				networkingv1.PolicyTypeIngress,
+				networkingv1.PolicyTypeEgress,
+			},
+			Ingress: []networkingv1.NetworkPolicyIngressRule{
+				{},
+			},
+			Egress: []networkingv1.NetworkPolicyEgressRule{
+				{
+					To: []networkingv1.NetworkPolicyPeer{
+						{
+							IPBlock: &networkingv1.IPBlock{
+								CIDR:   allTrafficCIDR,
+								Except: cidrList,
+							},
+						},
+						{
+							NamespaceSelector: &metav1.LabelSelector{
+								MatchLabels: map[string]string{
+									"kubernetes.io/metadata.name": clusterSet.Namespace,
+								},
+							},
+						},
+						{
+							NamespaceSelector: &metav1.LabelSelector{
+								MatchLabels: map[string]string{
+									"kubernetes.io/metadata.name": metav1.NamespaceSystem,
+								},
+							},
+							PodSelector: &metav1.LabelSelector{
+								MatchLabels: map[string]string{
+									"k8s-app": "kube-dns",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}, nil
+}
+
+func (c *ClusterSetReconciler) reconcileNamespacePodSecurityLabels(ctx context.Context, log *zap.SugaredLogger, clusterSet *v1alpha1.ClusterSet) error {
+	log.Info("reconciling Namespace")
+
+	var ns v1.Namespace
+	key := types.NamespacedName{Name: clusterSet.Namespace}
+	if err := c.Client.Get(ctx, key, &ns); err != nil {
+		return err
+	}
+
+	newLabels := map[string]string{}
+	for k, v := range ns.Labels {
+		newLabels[k] = v
+	}
+
+	// cleanup of old labels
+	delete(newLabels, "pod-security.kubernetes.io/enforce")
+	delete(newLabels, "pod-security.kubernetes.io/enforce-version")
+	delete(newLabels, "pod-security.kubernetes.io/warn")
+	delete(newLabels, "pod-security.kubernetes.io/warn-version")
+
+	// if a PSA level is specified add the proper labels
+	if clusterSet.Spec.PodSecurityAdmissionLevel != nil {
+		psaLevel := *clusterSet.Spec.PodSecurityAdmissionLevel
+
+		newLabels["pod-security.kubernetes.io/enforce"] = string(psaLevel)
+		newLabels["pod-security.kubernetes.io/enforce-version"] = "latest"
+
+		// skip the 'warn' only for the privileged PSA level
+		if psaLevel != v1alpha1.PrivilegedPodSecurityAdmissionLevel {
+			newLabels["pod-security.kubernetes.io/warn"] = string(psaLevel)
+			newLabels["pod-security.kubernetes.io/warn-version"] = "latest"
+		}
+	}
+
+	if !reflect.DeepEqual(ns.Labels, newLabels) {
+		log.Debug("labels changed, updating namespace")
+
+		ns.Labels = newLabels
+		return c.Client.Update(ctx, &ns)
+	}
+	return nil
+}
+
+func (c *ClusterSetReconciler) reconcileClusters(ctx context.Context, log *zap.SugaredLogger, clusterSet *v1alpha1.ClusterSet) error {
+	log.Info("reconciling Clusters")
+
+	var clusters v1alpha1.ClusterList
+	if err := c.Client.List(ctx, &clusters, ctrlruntimeclient.InNamespace(clusterSet.Namespace)); err != nil {
+		return err
+	}
+
+	var err error
+
+	for _, cluster := range clusters.Items {
+		oldClusterSpec := cluster.Spec
+
+		if cluster.Spec.PriorityClass != clusterSet.Spec.DefaultPriorityClass {
+			cluster.Spec.PriorityClass = clusterSet.Spec.DefaultPriorityClass
+		}
+
+		if !reflect.DeepEqual(cluster.Spec.NodeSelector, clusterSet.Spec.DefaultNodeSelector) {
+			cluster.Spec.NodeSelector = clusterSet.Spec.DefaultNodeSelector
+		}
+
+		if !reflect.DeepEqual(oldClusterSpec, cluster.Spec) {
+			// continue updating also the other clusters even if an error occurred
+			err = errors.Join(c.Client.Update(ctx, &cluster))
+		}
+	}
+
+	return err
+}

pkg/controller/clusterset/clusterset_suite_test.go

@@ -0,0 +1,88 @@
+package clusterset_test
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller/clusterset"
+	"github.com/rancher/k3k/pkg/log"
+
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestController(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ClusterSet Controller Suite")
+}
+
+var (
+	testEnv   *envtest.Environment
+	k8sClient client.Client
+	ctx       context.Context
+	cancel    context.CancelFunc
+)
+
+var _ = BeforeSuite(func() {
+
+	By("bootstrapping test environment")
+	testEnv = &envtest.Environment{
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")},
+		ErrorIfCRDPathMissing: true,
+	}
+	cfg, err := testEnv.Start()
+	Expect(err).NotTo(HaveOccurred())
+
+	scheme := buildScheme()
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
+	Expect(err).NotTo(HaveOccurred())
+
+	mgr, err := ctrl.NewManager(cfg, ctrl.Options{Scheme: scheme})
+	Expect(err).NotTo(HaveOccurred())
+
+	ctx, cancel = context.WithCancel(context.Background())
+	nopLogger := &log.Logger{SugaredLogger: zap.NewNop().Sugar()}
+
+	err = clusterset.Add(ctx, mgr, "", nopLogger)
+	Expect(err).NotTo(HaveOccurred())
+
+	go func() {
+		defer GinkgoRecover()
+		err = mgr.Start(ctx)
+		Expect(err).NotTo(HaveOccurred(), "failed to run manager")
+	}()
+})
+
+var _ = AfterSuite(func() {
+	cancel()
+
+	By("tearing down the test environment")
+	err := testEnv.Stop()
+	Expect(err).NotTo(HaveOccurred())
+})
+
+func buildScheme() *runtime.Scheme {
+	scheme := runtime.NewScheme()
+
+	err := corev1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+	err = appsv1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+	err = networkingv1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+	err = v1alpha1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	return scheme
+}

pkg/controller/clusterset/clusterset_test.go

@@ -0,0 +1,670 @@
+package clusterset_test
+
+import (
+	"context"
+	"reflect"
+	"time"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+
+	k3kcontroller "github.com/rancher/k3k/pkg/controller"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ClusterSet Controller", func() {
+
+	Context("creating a ClusterSet", func() {
+
+		var (
+			namespace string
+		)
+
+		BeforeEach(func() {
+			createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
+			err := k8sClient.Create(context.Background(), createdNS)
+			Expect(err).To(Not(HaveOccurred()))
+			namespace = createdNS.Name
+		})
+
+		When("created with a default spec", func() {
+			It("should have only the 'shared' allowedNodeTypes", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				allowedModeTypes := clusterSet.Spec.AllowedNodeTypes
+				Expect(allowedModeTypes).To(HaveLen(1))
+				Expect(allowedModeTypes).To(ContainElement(v1alpha1.SharedClusterMode))
+			})
+
+			It("should create a NetworkPolicy", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// look for network policies etc
+				clusterSetNetworkPolicy := &networkingv1.NetworkPolicy{}
+
+				Eventually(func() error {
+					key := types.NamespacedName{
+						Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+						Namespace: namespace,
+					}
+					return k8sClient.Get(ctx, key, clusterSetNetworkPolicy)
+				}).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeNil())
+
+				spec := clusterSetNetworkPolicy.Spec
+				Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeEgress))
+				Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeIngress))
+
+				// ingress should allow everything
+				Expect(spec.Ingress).To(ConsistOf(networkingv1.NetworkPolicyIngressRule{}))
+
+				// egress should contains some rules
+				Expect(spec.Egress).To(HaveLen(1))
+
+				// allow networking to all external IPs
+				ipBlockRule := networkingv1.NetworkPolicyPeer{
+					IPBlock: &networkingv1.IPBlock{CIDR: "0.0.0.0/0"},
+				}
+
+				// allow networking in the same namespace
+				clusterSetNamespaceRule := networkingv1.NetworkPolicyPeer{
+					NamespaceSelector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{"kubernetes.io/metadata.name": namespace},
+					},
+				}
+
+				// allow networking to the "kube-dns" pod in the "kube-system" namespace
+				kubeDNSRule := networkingv1.NetworkPolicyPeer{
+					PodSelector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{"k8s-app": "kube-dns"},
+					},
+					NamespaceSelector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"},
+					},
+				}
+
+				Expect(spec.Egress[0].To).To(ContainElements(
+					ipBlockRule, clusterSetNamespaceRule, kubeDNSRule,
+				))
+			})
+		})
+
+		When("created with DisableNetworkPolicy", func() {
+			It("should not create a NetworkPolicy if true", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						DisableNetworkPolicy: true,
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// wait for a bit for the network policy, but it should not be created
+				Eventually(func() bool {
+					key := types.NamespacedName{
+						Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+						Namespace: namespace,
+					}
+					err := k8sClient.Get(ctx, key, &networkingv1.NetworkPolicy{})
+					return apierrors.IsNotFound(err)
+				}).
+					MustPassRepeatedly(5).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+			})
+
+			It("should delete the NetworkPolicy if changed to false", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// look for network policy
+				clusterSetNetworkPolicy := &networkingv1.NetworkPolicy{}
+
+				Eventually(func() error {
+					key := types.NamespacedName{
+						Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+						Namespace: namespace,
+					}
+					return k8sClient.Get(ctx, key, clusterSetNetworkPolicy)
+				}).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeNil())
+
+				clusterSet.Spec.DisableNetworkPolicy = true
+				err = k8sClient.Update(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// wait for a bit for the network policy to being deleted
+				Eventually(func() bool {
+					key := types.NamespacedName{
+						Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+						Namespace: namespace,
+					}
+					err := k8sClient.Get(ctx, key, clusterSetNetworkPolicy)
+					return apierrors.IsNotFound(err)
+				}).
+					MustPassRepeatedly(5).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+			})
+
+			It("should recreate the NetworkPolicy if deleted", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// look for network policy
+				clusterSetNetworkPolicy := &networkingv1.NetworkPolicy{}
+
+				Eventually(func() error {
+					key := types.NamespacedName{
+						Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+						Namespace: namespace,
+					}
+					return k8sClient.Get(context.Background(), key, clusterSetNetworkPolicy)
+				}).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeNil())
+
+				err = k8sClient.Delete(ctx, clusterSetNetworkPolicy)
+				Expect(err).To(Not(HaveOccurred()))
+
+				key := types.NamespacedName{
+					Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+					Namespace: namespace,
+				}
+				err = k8sClient.Get(ctx, key, clusterSetNetworkPolicy)
+				Expect(apierrors.IsNotFound(err)).Should(BeTrue())
+
+				// wait a bit for the network policy to being recreated
+				Eventually(func() error {
+					key := types.NamespacedName{
+						Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+						Namespace: namespace,
+					}
+					return k8sClient.Get(ctx, key, clusterSetNetworkPolicy)
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeNil())
+			})
+
+		})
+
+		When("created specifing the mode", func() {
+			It("should have the 'virtual' mode if specified", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						AllowedNodeTypes: []v1alpha1.ClusterMode{
+							v1alpha1.VirtualClusterMode,
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				allowedModeTypes := clusterSet.Spec.AllowedNodeTypes
+				Expect(allowedModeTypes).To(HaveLen(1))
+				Expect(allowedModeTypes).To(ContainElement(v1alpha1.VirtualClusterMode))
+			})
+
+			It("should have both modes if specified", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						AllowedNodeTypes: []v1alpha1.ClusterMode{
+							v1alpha1.SharedClusterMode,
+							v1alpha1.VirtualClusterMode,
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				allowedModeTypes := clusterSet.Spec.AllowedNodeTypes
+				Expect(allowedModeTypes).To(HaveLen(2))
+				Expect(allowedModeTypes).To(ContainElements(
+					v1alpha1.SharedClusterMode,
+					v1alpha1.VirtualClusterMode,
+				))
+			})
+
+			It("should fail for a non-existing mode", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						AllowedNodeTypes: []v1alpha1.ClusterMode{
+							v1alpha1.SharedClusterMode,
+							v1alpha1.VirtualClusterMode,
+							v1alpha1.ClusterMode("non-existing"),
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(HaveOccurred())
+			})
+		})
+
+		When("created specifing the podSecurityAdmissionLevel", func() {
+			It("should add and update the proper pod-security labels to the namespace", func() {
+				var (
+					privileged = v1alpha1.PrivilegedPodSecurityAdmissionLevel
+					baseline   = v1alpha1.BaselinePodSecurityAdmissionLevel
+					restricted = v1alpha1.RestrictedPodSecurityAdmissionLevel
+				)
+
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						PodSecurityAdmissionLevel: &privileged,
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				var ns corev1.Namespace
+
+				// Check privileged
+
+				// wait a bit for the namespace to be updated
+				Eventually(func() bool {
+					err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace}, &ns)
+					Expect(err).To(Not(HaveOccurred()))
+					enforceValue := ns.Labels["pod-security.kubernetes.io/enforce"]
+					return enforceValue == "privileged"
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "privileged"))
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
+				Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/warn")))
+				Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/warn-version")))
+
+				// Check baseline
+
+				clusterSet.Spec.PodSecurityAdmissionLevel = &baseline
+				err = k8sClient.Update(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// wait a bit for the namespace to be updated
+				Eventually(func() bool {
+					err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace}, &ns)
+					Expect(err).To(Not(HaveOccurred()))
+					enforceValue := ns.Labels["pod-security.kubernetes.io/enforce"]
+					return enforceValue == "baseline"
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "baseline"))
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/warn", "baseline"))
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/warn-version", "latest"))
+
+				// Check restricted
+
+				clusterSet.Spec.PodSecurityAdmissionLevel = &restricted
+				err = k8sClient.Update(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// wait a bit for the namespace to be updated
+				Eventually(func() bool {
+					err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace}, &ns)
+					Expect(err).To(Not(HaveOccurred()))
+					enforceValue := ns.Labels["pod-security.kubernetes.io/enforce"]
+					return enforceValue == "restricted"
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "restricted"))
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/warn", "restricted"))
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/warn-version", "latest"))
+
+				// check cleanup
+
+				clusterSet.Spec.PodSecurityAdmissionLevel = nil
+				err = k8sClient.Update(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// wait a bit for the namespace to be updated
+				Eventually(func() bool {
+					err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace}, &ns)
+					Expect(err).To(Not(HaveOccurred()))
+					_, found := ns.Labels["pod-security.kubernetes.io/enforce"]
+					return found
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeFalse())
+
+				Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/enforce")))
+				Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/enforce-version")))
+				Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/warn")))
+				Expect(ns.Labels).Should(Not(HaveKey("pod-security.kubernetes.io/warn-version")))
+			})
+
+			It("should restore the labels if Namespace is updated", func() {
+				privileged := v1alpha1.PrivilegedPodSecurityAdmissionLevel
+
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						PodSecurityAdmissionLevel: &privileged,
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				var ns corev1.Namespace
+
+				// wait a bit for the namespace to be updated
+				Eventually(func() bool {
+					err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace}, &ns)
+					Expect(err).To(Not(HaveOccurred()))
+					enforceValue := ns.Labels["pod-security.kubernetes.io/enforce"]
+					return enforceValue == "privileged"
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "privileged"))
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
+
+				ns.Labels["pod-security.kubernetes.io/enforce"] = "baseline"
+				err = k8sClient.Update(ctx, &ns)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// wait a bit for the namespace to be restored
+				Eventually(func() bool {
+					err = k8sClient.Get(ctx, types.NamespacedName{Name: namespace}, &ns)
+					Expect(err).To(Not(HaveOccurred()))
+					enforceValue := ns.Labels["pod-security.kubernetes.io/enforce"]
+					return enforceValue == "privileged"
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce", "privileged"))
+				Expect(ns.Labels).Should(HaveKeyWithValue("pod-security.kubernetes.io/enforce-version", "latest"))
+			})
+		})
+
+		When("a cluster in the same namespace is present", func() {
+			It("should update it if needed", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						DefaultPriorityClass: "foobar",
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				cluster := &v1alpha1.Cluster{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "cluster-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSpec{
+						Mode:    v1alpha1.SharedClusterMode,
+						Servers: ptr.To(int32(1)),
+						Agents:  ptr.To(int32(0)),
+					},
+				}
+
+				err = k8sClient.Create(ctx, cluster)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// wait a bit
+				Eventually(func() bool {
+					key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
+					err = k8sClient.Get(ctx, key, cluster)
+					Expect(err).To(Not(HaveOccurred()))
+					return cluster.Spec.PriorityClass == clusterSet.Spec.DefaultPriorityClass
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+			})
+
+			It("should update the nodeSelector", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						DefaultNodeSelector: map[string]string{"label-1": "value-1"},
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				cluster := &v1alpha1.Cluster{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "cluster-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSpec{
+						Mode:    v1alpha1.SharedClusterMode,
+						Servers: ptr.To(int32(1)),
+						Agents:  ptr.To(int32(0)),
+					},
+				}
+
+				err = k8sClient.Create(ctx, cluster)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// wait a bit
+				Eventually(func() bool {
+					key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
+					err = k8sClient.Get(ctx, key, cluster)
+					Expect(err).To(Not(HaveOccurred()))
+					return reflect.DeepEqual(cluster.Spec.NodeSelector, clusterSet.Spec.DefaultNodeSelector)
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+			})
+
+			It("should update the nodeSelector if changed", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						DefaultNodeSelector: map[string]string{"label-1": "value-1"},
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				cluster := &v1alpha1.Cluster{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "cluster-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSpec{
+						Mode:         v1alpha1.SharedClusterMode,
+						Servers:      ptr.To(int32(1)),
+						Agents:       ptr.To(int32(0)),
+						NodeSelector: map[string]string{"label-1": "value-1"},
+					},
+				}
+
+				err = k8sClient.Create(ctx, cluster)
+				Expect(err).To(Not(HaveOccurred()))
+
+				Expect(cluster.Spec.NodeSelector).To(Equal(clusterSet.Spec.DefaultNodeSelector))
+
+				// update the ClusterSet
+				clusterSet.Spec.DefaultNodeSelector["label-2"] = "value-2"
+				err = k8sClient.Update(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+				Expect(cluster.Spec.NodeSelector).To(Not(Equal(clusterSet.Spec.DefaultNodeSelector)))
+
+				// wait a bit
+				Eventually(func() bool {
+					key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
+					err = k8sClient.Get(ctx, key, cluster)
+					Expect(err).To(Not(HaveOccurred()))
+					return reflect.DeepEqual(cluster.Spec.NodeSelector, clusterSet.Spec.DefaultNodeSelector)
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+
+				// Update the Cluster
+				cluster.Spec.NodeSelector["label-3"] = "value-3"
+				err = k8sClient.Update(ctx, cluster)
+				Expect(err).To(Not(HaveOccurred()))
+				Expect(cluster.Spec.NodeSelector).To(Not(Equal(clusterSet.Spec.DefaultNodeSelector)))
+
+				// wait a bit and check it's restored
+				Eventually(func() bool {
+					var updatedCluster v1alpha1.Cluster
+
+					key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
+					err = k8sClient.Get(ctx, key, &updatedCluster)
+					Expect(err).To(Not(HaveOccurred()))
+					return reflect.DeepEqual(updatedCluster.Spec.NodeSelector, clusterSet.Spec.DefaultNodeSelector)
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+			})
+		})
+
+		When("a cluster in a different namespace is present", func() {
+			It("should not be update", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						DefaultPriorityClass: "foobar",
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				namespace2 := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
+				err = k8sClient.Create(ctx, namespace2)
+				Expect(err).To(Not(HaveOccurred()))
+
+				cluster := &v1alpha1.Cluster{
+					ObjectMeta: v1.ObjectMeta{
+						GenerateName: "cluster-",
+						Namespace:    namespace2.Name,
+					},
+					Spec: v1alpha1.ClusterSpec{
+						Mode:    v1alpha1.SharedClusterMode,
+						Servers: ptr.To(int32(1)),
+						Agents:  ptr.To(int32(0)),
+					},
+				}
+
+				err = k8sClient.Create(ctx, cluster)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// it should not change!
+				Eventually(func() bool {
+					key := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
+					err = k8sClient.Get(ctx, key, cluster)
+					Expect(err).To(Not(HaveOccurred()))
+					return cluster.Spec.PriorityClass != clusterSet.Spec.DefaultPriorityClass
+				}).
+					MustPassRepeatedly(5).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+			})
+		})
+	})
+})

pkg/controller/clusterset/node.go

@@ -0,0 +1,84 @@
+package clusterset
+
+import (
+	"context"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/log"
+	"go.uber.org/zap"
+	v1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+	nodeController = "k3k-node-controller"
+)
+
+type NodeReconciler struct {
+	Client      ctrlruntimeclient.Client
+	Scheme      *runtime.Scheme
+	ClusterCIDR string
+	logger      *log.Logger
+}
+
+// AddNodeController adds a new controller to the manager
+func AddNodeController(ctx context.Context, mgr manager.Manager, logger *log.Logger) error {
+	// initialize a new Reconciler
+	reconciler := NodeReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+		logger: logger.Named(nodeController),
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1.Node{}).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: maxConcurrentReconciles,
+		}).
+		Named(nodeController).
+		Complete(&reconciler)
+}
+
+func (n *NodeReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	log := n.logger.With("Node", req.NamespacedName)
+	var clusterSetList v1alpha1.ClusterSetList
+	if err := n.Client.List(ctx, &clusterSetList); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	if len(clusterSetList.Items) <= 0 {
+		return reconcile.Result{}, nil
+	}
+
+	if err := n.ensureNetworkPolicies(ctx, clusterSetList, log); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	return reconcile.Result{}, nil
+}
+
+func (n *NodeReconciler) ensureNetworkPolicies(ctx context.Context, clusterSetList v1alpha1.ClusterSetList, log *zap.SugaredLogger) error {
+	var setNetworkPolicy *networkingv1.NetworkPolicy
+	for _, cs := range clusterSetList.Items {
+		if cs.Spec.DisableNetworkPolicy {
+			continue
+		}
+		var err error
+		log.Infow("Updating NetworkPolicy for ClusterSet", "name", cs.Name, "namespace", cs.Namespace)
+		setNetworkPolicy, err = netpol(ctx, "", &cs, n.Client)
+		if err != nil {
+			return err
+		}
+		log.Debugw("New NetworkPolicy for clusterset", "name", cs.Name, "namespace", cs.Namespace)
+		if err := n.Client.Update(ctx, setNetworkPolicy); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/controller/controller.go

@@ -0,0 +1,89 @@
+package controller
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+	"time"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	namePrefix      = "k3k"
+	k3SImageName    = "rancher/k3s"
+	AdminCommonName = "system:admin"
+)
+
+// Backoff is the cluster creation duration backoff
+var Backoff = wait.Backoff{
+	Steps:    5,
+	Duration: 5 * time.Second,
+	Factor:   2,
+	Jitter:   0.1,
+}
+
+func K3SImage(cluster *v1alpha1.Cluster) string {
+	return k3SImageName + ":" + cluster.Spec.Version
+}
+func nodeAddress(node *v1.Node) string {
+	var externalIP string
+	var internalIP string
+
+	for _, ip := range node.Status.Addresses {
+		if ip.Type == "ExternalIP" && ip.Address != "" {
+			externalIP = ip.Address
+			break
+		}
+		if ip.Type == "InternalIP" && ip.Address != "" {
+			internalIP = ip.Address
+		}
+	}
+	if externalIP != "" {
+		return externalIP
+	}
+
+	return internalIP
+}
+
+// return all the nodes external addresses, if not found then return internal addresses
+func Addresses(ctx context.Context, client ctrlruntimeclient.Client) ([]string, error) {
+	var nodeList v1.NodeList
+	if err := client.List(ctx, &nodeList); err != nil {
+		return nil, err
+	}
+
+	addresses := make([]string, len(nodeList.Items))
+	for _, node := range nodeList.Items {
+		addresses = append(addresses, nodeAddress(&node))
+	}
+
+	return addresses, nil
+}
+
+// SafeConcatNameWithPrefix runs the SafeConcatName with extra prefix.
+func SafeConcatNameWithPrefix(name ...string) string {
+	return SafeConcatName(append([]string{namePrefix}, name...)...)
+}
+
+// SafeConcatName concatenates the given strings and ensures the returned name is under 64 characters
+// by cutting the string off at 57 characters and setting the last 6 with an encoded version of the concatenated string.
+func SafeConcatName(name ...string) string {
+	fullPath := strings.Join(name, "-")
+	if len(fullPath) < 64 {
+		return fullPath
+	}
+	digest := sha256.Sum256([]byte(fullPath))
+	// since we cut the string in the middle, the last char may not be compatible with what is expected in k8s
+	// we are checking and if necessary removing the last char
+	c := fullPath[56]
+	if 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
+		return fullPath[0:57] + "-" + hex.EncodeToString(digest[0:])[0:5]
+	}
+
+	return fullPath[0:56] + "-" + hex.EncodeToString(digest[0:])[0:6]
+}

pkg/controller/kubeconfig/kubeconfig.go

@@ -0,0 +1,110 @@
+package kubeconfig
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	certutil "github.com/rancher/dynamiclistener/cert"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/certs"
+	"github.com/rancher/k3k/pkg/controller/cluster/server"
+	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type KubeConfig struct {
+	AltNames   certutil.AltNames
+	CN         string
+	ORG        []string
+	ExpiryDate time.Duration
+}
+
+func (k *KubeConfig) Extract(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, hostServerIP string) ([]byte, error) {
+	nn := types.NamespacedName{
+		Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
+		Namespace: cluster.Namespace,
+	}
+
+	var bootstrapSecret v1.Secret
+	if err := client.Get(ctx, nn, &bootstrapSecret); err != nil {
+		return nil, err
+	}
+
+	bootstrapData := bootstrapSecret.Data["bootstrap"]
+	if bootstrapData == nil {
+		return nil, errors.New("empty bootstrap")
+	}
+
+	var bootstrap bootstrap.ControlRuntimeBootstrap
+	if err := json.Unmarshal(bootstrapData, &bootstrap); err != nil {
+		return nil, err
+	}
+
+	adminCert, adminKey, err := certs.CreateClientCertKey(
+		k.CN, k.ORG,
+		&k.AltNames, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, k.ExpiryDate,
+		bootstrap.ClientCA.Content,
+		bootstrap.ClientCAKey.Content)
+	if err != nil {
+		return nil, err
+	}
+	// get the server service to extract the right IP
+	nn = types.NamespacedName{
+		Name:      server.ServiceName(cluster.Name),
+		Namespace: cluster.Namespace,
+	}
+
+	var k3kService v1.Service
+	if err := client.Get(ctx, nn, &k3kService); err != nil {
+		return nil, err
+	}
+
+	url := fmt.Sprintf("https://%s:%d", k3kService.Spec.ClusterIP, server.ServerPort)
+	if k3kService.Spec.Type == v1.ServiceTypeNodePort {
+		nodePort := k3kService.Spec.Ports[0].NodePort
+		url = fmt.Sprintf("https://%s:%d", hostServerIP, nodePort)
+	}
+	kubeconfigData, err := kubeconfig(url, []byte(bootstrap.ServerCA.Content), adminCert, adminKey)
+	if err != nil {
+		return nil, err
+	}
+
+	return kubeconfigData, nil
+}
+
+func kubeconfig(url string, serverCA, clientCert, clientKey []byte) ([]byte, error) {
+	config := clientcmdapi.NewConfig()
+
+	cluster := clientcmdapi.NewCluster()
+	cluster.CertificateAuthorityData = serverCA
+	cluster.Server = url
+
+	authInfo := clientcmdapi.NewAuthInfo()
+	authInfo.ClientCertificateData = clientCert
+	authInfo.ClientKeyData = clientKey
+
+	context := clientcmdapi.NewContext()
+	context.AuthInfo = "default"
+	context.Cluster = "default"
+
+	config.Clusters["default"] = cluster
+	config.AuthInfos["default"] = authInfo
+	config.Contexts["default"] = context
+	config.CurrentContext = "default"
+
+	kubeconfig, err := clientcmd.Write(*config)
+	if err != nil {
+		return nil, err
+	}
+
+	return kubeconfig, nil
+}

pkg/controller/util/util.go

@@ -1,67 +0,0 @@
-package util
-
-import (
-	"context"
-
-	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/klog"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-)
-
-const (
-	namespacePrefix = "k3k-"
-	k3SImageName    = "rancher/k3s"
-)
-
-const (
-	K3kSystemNamespace = namespacePrefix + "system"
-)
-
-func ClusterNamespace(cluster *v1alpha1.Cluster) string {
-	return namespacePrefix + cluster.Name
-}
-
-func K3SImage(cluster *v1alpha1.Cluster) string {
-	return k3SImageName + ":" + cluster.Spec.Version
-}
-
-func LogAndReturnErr(errString string, err error) error {
-	klog.Errorf("%s: %v", errString, err)
-	return err
-}
-
-func nodeAddress(node *v1.Node) string {
-	var externalIP string
-	var internalIP string
-
-	for _, ip := range node.Status.Addresses {
-		if ip.Type == "ExternalIP" && ip.Address != "" {
-			externalIP = ip.Address
-			break
-		}
-		if ip.Type == "InternalIP" && ip.Address != "" {
-			internalIP = ip.Address
-		}
-	}
-	if externalIP != "" {
-		return externalIP
-	}
-
-	return internalIP
-}
-
-// return all the nodes external addresses, if not found then return internal addresses
-func Addresses(ctx context.Context, client client.Client) ([]string, error) {
-	var nodeList v1.NodeList
-	if err := client.List(ctx, &nodeList); err != nil {
-		return nil, err
-	}
-
-	var addresses []string
-	for _, node := range nodeList.Items {
-		addresses = append(addresses, nodeAddress(&node))
-	}
-
-	return addresses, nil
-}

pkg/log/zap.go

@@ -0,0 +1,51 @@
+package log
+
+import (
+	"os"
+
+	"github.com/virtual-kubelet/virtual-kubelet/log"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	ctrlruntimezap "sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+type Logger struct {
+	*zap.SugaredLogger
+}
+
+func New(debug bool) *Logger {
+	return &Logger{newZappLogger(debug).Sugar()}
+}
+
+func (l *Logger) WithError(err error) log.Logger {
+	return l
+}
+
+func (l *Logger) WithField(string, interface{}) log.Logger {
+	return l
+}
+
+func (l *Logger) WithFields(field log.Fields) log.Logger {
+	return l
+}
+
+func (l *Logger) Named(name string) *Logger {
+	l.SugaredLogger = l.SugaredLogger.Named(name)
+	return l
+}
+
+func newZappLogger(debug bool) *zap.Logger {
+	encCfg := zap.NewProductionEncoderConfig()
+	encCfg.TimeKey = "timestamp"
+	encCfg.EncodeTime = zapcore.ISO8601TimeEncoder
+
+	lvl := zap.NewAtomicLevelAt(zap.InfoLevel)
+	if debug {
+		lvl = zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+
+	encoder := zapcore.NewJSONEncoder(encCfg)
+	core := zapcore.NewCore(&ctrlruntimezap.KubeAwareEncoder{Encoder: encoder}, zapcore.AddSync(os.Stderr), lvl)
+
+	return zap.New(core)
+}