Add pods/attach permission to k3k-kubelet (#400)

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

.github/workflows/test-conformance.yaml

@@ -0,0 +1,296 @@
+name: Conformance Tests
+
+on:
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+    inputs:
+      test:
+        description: "Run specific test"
+        type: choice
+        options:
+          - conformance
+          - sig-api-machinery
+          - sig-apps
+          - sig-architecture
+          - sig-auth
+          - sig-cli
+          - sig-instrumentation
+          - sig-network
+          - sig-node
+          - sig-scheduling
+          - sig-storage
+
+permissions:
+    contents: read
+
+jobs:
+  conformance:
+    runs-on: ubuntu-latest
+    if: inputs.test == '' || inputs.test == 'conformance'
+
+    strategy:
+      fail-fast: false
+      matrix:
+        type:
+        - parallel
+        - serial
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Install helm
+      uses: azure/setup-helm@v4.3.0
+    
+    - name: Install hydrophone
+      run: go install sigs.k8s.io/hydrophone@latest
+
+    - name: Install k3d and kubectl
+      run: |
+        wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+        k3d version
+
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+
+    - name: Setup Kubernetes (k3d)
+      env:
+        REPO_NAME: k3k-registry
+        REPO_PORT: 12345
+      run: |
+        echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts
+
+        k3d registry create ${REPO_NAME} --port ${REPO_PORT}
+        
+        k3d cluster create k3k --servers 3 \
+          -p "30000-30010:30000-30010@server:0" \
+          --registry-use k3d-${REPO_NAME}:${REPO_PORT}
+        
+        kubectl cluster-info
+        kubectl get nodes
+
+    - name: Setup K3k
+      env:
+        REPO: k3k-registry:12345
+      run: |
+        echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts
+
+        make build
+        make package
+        make push
+
+        # add k3kcli to $PATH
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+        VERSION=$(make version)
+        k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose
+        k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose
+
+        make install
+        
+        echo "Wait for K3k controller to be available"
+        kubectl wait -n k3k-system pod --for condition=Ready -l "app.kubernetes.io/name=k3k" --timeout=5m
+
+    - name: Check k3kcli
+      run: k3kcli -v
+
+    - name: Create virtual cluster
+      run: |
+        kubectl create namespace k3k-mycluster
+
+        cat <<EOF | kubectl apply -f -
+        apiVersion: k3k.io/v1alpha1
+        kind: Cluster
+        metadata:
+          name: mycluster
+          namespace: k3k-mycluster
+        spec:
+          servers: 2
+          tlsSANs:
+            - "127.0.0.1"
+          expose:
+            nodePort:
+              serverPort: 30001
+        EOF
+
+        echo "Wait for bootstrap secret to be available"
+        kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m
+
+        k3kcli kubeconfig generate --name mycluster
+
+        export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
+        
+        kubectl cluster-info
+        kubectl get nodes
+        kubectl get pods -A
+    
+    - name: Run conformance tests (parallel)
+      if: matrix.type == 'parallel'
+      run: |
+        # Run conformance tests in parallel mode (skipping serial)
+        hydrophone --conformance --parallel 4 --skip='\[Serial\]' \
+          --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
+          --output-dir /tmp
+
+    - name: Run conformance tests (serial)
+      if: matrix.type == 'serial'
+      run: |        
+        # Run serial conformance tests
+        hydrophone --focus='\[Serial\].*\[Conformance\]' \
+          --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
+          --output-dir /tmp 
+
+    - name: Archive conformance logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: conformance-${{ matrix.type }}-logs
+        path: /tmp/e2e.log
+
+  sigs:
+    runs-on: ubuntu-latest
+    if: inputs.test == '' || inputs.test != 'conformance'
+
+    strategy:
+      fail-fast: false
+      matrix:
+        tests:
+          - name: sig-api-machinery
+            focus: '\[sig-api-machinery\].*\[Conformance\]'
+          - name: sig-apps
+            focus: '\[sig-apps\].*\[Conformance\]'
+          - name: sig-architecture
+            focus: '\[sig-architecture\].*\[Conformance\]'
+          - name: sig-auth
+            focus: '\[sig-auth\].*\[Conformance\]'
+          - name: sig-cli
+            focus: '\[sig-cli\].*\[Conformance\]'
+          - name: sig-instrumentation
+            focus: '\[sig-instrumentation\].*\[Conformance\]'
+          - name: sig-network
+            focus: '\[sig-network\].*\[Conformance\]'
+          - name: sig-node
+            focus: '\[sig-node\].*\[Conformance\]'
+          - name: sig-scheduling
+            focus: '\[sig-scheduling\].*\[Conformance\]'
+          - name: sig-storage
+            focus: '\[sig-storage\].*\[Conformance\]'
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Install helm
+      uses: azure/setup-helm@v4.3.0
+    
+    - name: Install hydrophone
+      run: go install sigs.k8s.io/hydrophone@latest
+
+    - name: Install k3d and kubectl
+      run: |
+        wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+        k3d version
+
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+
+    - name: Setup Kubernetes (k3d)
+      env:
+        REPO_NAME: k3k-registry
+        REPO_PORT: 12345
+      run: |
+        echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts
+
+        k3d registry create ${REPO_NAME} --port ${REPO_PORT}
+        
+        k3d cluster create k3k --servers 3 \
+          -p "30000-30010:30000-30010@server:0" \
+          --registry-use k3d-${REPO_NAME}:${REPO_PORT}
+        
+        kubectl cluster-info
+        kubectl get nodes
+
+    - name: Setup K3k
+      env:
+        REPO: k3k-registry:12345
+      run: |
+        echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts
+
+        make build
+        make package
+        make push
+
+        # add k3kcli to $PATH
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+        VERSION=$(make version)
+        k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose
+        k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose
+
+        make install
+        
+        echo "Wait for K3k controller to be available"
+        kubectl wait -n k3k-system pod --for condition=Ready -l "app.kubernetes.io/name=k3k" --timeout=5m
+
+    - name: Check k3kcli
+      run: k3kcli -v
+
+    - name: Create virtual cluster
+      run: |
+        kubectl create namespace k3k-mycluster
+
+        cat <<EOF | kubectl apply -f -
+        apiVersion: k3k.io/v1alpha1
+        kind: Cluster
+        metadata:
+          name: mycluster
+          namespace: k3k-mycluster
+        spec:
+          servers: 2
+          tlsSANs:
+            - "127.0.0.1"
+          expose:
+            nodePort:
+              serverPort: 30001
+        EOF
+
+        echo "Wait for bootstrap secret to be available"
+        kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m
+
+        k3kcli kubeconfig generate --name mycluster
+
+        export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
+        
+        kubectl cluster-info
+        kubectl get nodes
+        kubectl get pods -A
+
+    - name: Run sigs tests
+      if: inputs.test == '' || inputs.test == matrix.tests.name
+      run: |
+        FOCUS="${{ matrix.tests.focus }}"
+        echo "Running with --focus=${FOCUS}"
+
+        hydrophone --focus "${FOCUS}" \
+          --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
+          --output-dir /tmp
+
+    - name: Archive conformance logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: ${{ matrix.tests.name }}-logs
+        path: /tmp/e2e.log

charts/k3k/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: k3k
 description: A Helm chart for K3K
 type: application
-version: 0.3.3-r3
-appVersion: v0.3.3-rc3
+version: 0.3.3-r5
+appVersion: v0.3.3-rc5

cli/cmds/cluster_create.go

@@ -142,7 +142,7 @@ func createAction(appCtx *AppContext, config *CreateConfig) cli.ActionFunc {
 		var kubeconfig *clientcmdapi.Config
 
 		if err := retry.OnError(availableBackoff, apierrors.IsNotFound, func() error {
-			kubeconfig, err = cfg.Extract(ctx, client, cluster, host[0])
+			kubeconfig, err = cfg.Generate(ctx, client, cluster, host[0])
 			return err
 		}); err != nil {
 			return err

cli/cmds/kubeconfig.go

@@ -148,7 +148,7 @@ func generate(appCtx *AppContext) cli.ActionFunc {
 		var kubeconfig *clientcmdapi.Config
 
 		if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
-			kubeconfig, err = cfg.Extract(ctx, client, &cluster, host[0])
+			kubeconfig, err = cfg.Generate(ctx, client, &cluster, host[0])
 			return err
 		}); err != nil {
 			return err

k3k-kubelet/controller/configmap.go

@@ -16,6 +16,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
+const ConfigMapSyncerName = "configmap-syncer"
+
 type ConfigMapSyncer struct {
 	mutex sync.RWMutex
 	// VirtualClient is the client for the virtual cluster
@@ -32,6 +34,10 @@ type ConfigMapSyncer struct {
 	objs sets.Set[types.NamespacedName]
 }
 
+func (c *ConfigMapSyncer) Name() string {
+	return ConfigMapSyncerName
+}
+
 // Reconcile implements reconcile.Reconciler and synchronizes the objects in objs to the host cluster
 func (c *ConfigMapSyncer) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
 	if !c.isWatching(req.NamespacedName) {

k3k-kubelet/controller/handler.go

@@ -39,6 +39,7 @@ type ControllerHandler struct {
 // be altered through the Add and Remove methods
 type updateableReconciler interface {
 	reconcile.Reconciler
+	Name() string
 	AddResource(ctx context.Context, namespace string, name string) error
 	RemoveResource(ctx context.Context, namespace string, name string) error
 }
@@ -97,6 +98,7 @@ func (c *ControllerHandler) AddResource(ctx context.Context, obj client.Object)
 	}
 
 	err := ctrl.NewControllerManagedBy(c.Mgr).
+		Named(r.Name()).
 		For(&v1.ConfigMap{}).
 		Complete(r)
 

k3k-kubelet/controller/secret.go

@@ -16,6 +16,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
+const SecretSyncerName = "secret-syncer"
+
 type SecretSyncer struct {
 	mutex sync.RWMutex
 	// VirtualClient is the client for the virtual cluster
@@ -32,6 +34,10 @@ type SecretSyncer struct {
 	objs sets.Set[types.NamespacedName]
 }
 
+func (s *SecretSyncer) Name() string {
+	return SecretSyncerName
+}
+
 // Reconcile implements reconcile.Reconciler and synchronizes the objects in objs to the host cluster
 func (s *SecretSyncer) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
 	if !s.isWatching(req.NamespacedName) {

pkg/controller/cluster/agent/shared.go

@@ -336,7 +336,7 @@ func (s *SharedAgent) role(ctx context.Context) error {
 		Rules: []rbacv1.PolicyRule{
 			{
 				APIGroups: []string{""},
-				Resources: []string{"persistentvolumeclaims", "pods", "pods/log", "pods/exec", "pods/ephemeralcontainers", "secrets", "configmaps", "services"},
+				Resources: []string{"persistentvolumeclaims", "pods", "pods/log", "pods/attach", "pods/exec", "pods/ephemeralcontainers", "secrets", "configmaps", "services"},
 				Verbs:     []string{"*"},
 			},
 			{

pkg/controller/cluster/cluster.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rancher/k3k/pkg/controller/cluster/agent"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	"github.com/rancher/k3k/pkg/controller/kubeconfig"
 	"github.com/rancher/k3k/pkg/controller/policy"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -25,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -289,6 +291,10 @@ func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alp
 		return err
 	}
 
+	if err := c.ensureKubeconfigSecret(ctx, cluster, serviceIP, token); err != nil {
+		return err
+	}
+
 	return c.bindNodeProxyClusterRole(ctx, cluster)
 }
 
@@ -324,6 +330,45 @@ func (c *ClusterReconciler) ensureBootstrapSecret(ctx context.Context, cluster *
 	return err
 }
 
+// ensureKubeconfigSecret will create or update the Secret containing the kubeconfig data from the k3s server
+func (c *ClusterReconciler) ensureKubeconfigSecret(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP, token string) error {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("ensuring kubeconfig secret")
+
+	adminKubeconfig := kubeconfig.New()
+
+	kubeconfig, err := adminKubeconfig.Generate(ctx, c.Client, cluster, serviceIP)
+	if err != nil {
+		return err
+	}
+
+	kubeconfigData, err := clientcmd.Write(*kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	kubeconfigSecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "kubeconfig"),
+			Namespace: cluster.Namespace,
+		},
+	}
+
+	_, err = controllerutil.CreateOrUpdate(ctx, c.Client, kubeconfigSecret, func() error {
+		if err := controllerutil.SetControllerReference(cluster, kubeconfigSecret, c.Scheme); err != nil {
+			return err
+		}
+
+		kubeconfigSecret.Data = map[string][]byte{
+			"kubeconfig.yaml": kubeconfigData,
+		}
+
+		return nil
+	})
+
+	return err
+}
+
 func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server, serviceIP string) error {
 	// create init node config
 	initServerConfig, err := server.Config(true, serviceIP)

pkg/controller/kubeconfig/kubeconfig.go

@@ -37,7 +37,7 @@ func New() *KubeConfig {
 	}
 }
 
-func (k *KubeConfig) Extract(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, hostServerIP string) (*clientcmdapi.Config, error) {
+func (k *KubeConfig) Generate(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, hostServerIP string) (*clientcmdapi.Config, error) {
 	bootstrapData, err := bootstrap.GetFromSecret(ctx, client, cluster)
 	if err != nil {
 		return nil, err

tests/common_test.go

@@ -191,7 +191,7 @@ func NewVirtualK8sClientAndConfig(cluster *v1alpha1.Cluster) (*kubernetes.Client
 		vKubeconfig := kubeconfig.New()
 		kubeletAltName := fmt.Sprintf("k3k-%s-kubelet", cluster.Name)
 		vKubeconfig.AltNames = certs.AddSANs([]string{hostIP, kubeletAltName})
-		config, err = vKubeconfig.Extract(ctx, k8sClient, cluster, hostIP)
+		config, err = vKubeconfig.Generate(ctx, k8sClient, cluster, hostIP)
 		return err
 	}).
 		WithTimeout(time.Minute * 2).