github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-19 04:16:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implementation of GetStatsSummary and GetMetricsResource for Virtual Kubelet (#163)

Browse Source 
* implemented  GetStatsSummary and GetMetricsResource for Virtual Kubelet

* fixed ClusterRole for node proxy

* limit the clusterrole with get and list

* remove unused Metrics client interface
This commit is contained in:
Enrico Candino
2024-12-27 11:41:40 +01:00
committed by GitHub
parent 70a098df4c
commit 7fdd48d577
7 changed files with 407 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/cluster/agent/shared.go
+25 -15
View File
@@ -16,7 +16,7 @@ import (
const (
sharedKubeletConfigPath = "/opt/rancher/k3k/config.yaml"
sharedNodeAgentName = "kubelet"
SharedNodeAgentName = "kubelet"
SharedNodeMode = "shared"
)
@@ -65,43 +65,50 @@ token: %s`,
}
func (s *SharedAgent) Resources() []ctrlruntimeclient.Object {
return []ctrlruntimeclient.Object{s.serviceAccount(), s.role(), s.roleBinding(), s.service(), s.deployment(), s.dnsService()}
return []ctrlruntimeclient.Object{
s.serviceAccount(),
s.role(),
s.roleBinding(),
s.service(),
s.deployment(),
s.dnsService(),
}
}
func (s *SharedAgent) deployment() *apps.Deployment {
selector := metav1.LabelSelector{
selector := &metav1.LabelSelector{
MatchLabels: map[string]string{
"cluster": s.cluster.Name,
"type": "agent",
"mode": "shared",
},
}
name := s.Name()
return &apps.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
Name: s.Name(),
Namespace: s.cluster.Namespace,
Labels: selector.MatchLabels,
},
Spec: apps.DeploymentSpec{
Selector: &selector,
Selector: selector,
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: selector.MatchLabels,
},
Spec: s.podSpec(s.sharedAgentImage, name, &selector),
Spec: s.podSpec(selector),
},
},
}
}
func (s *SharedAgent) podSpec(image, name string, affinitySelector *metav1.LabelSelector) v1.PodSpec {
args := []string{"--config", sharedKubeletConfigPath}
func (s *SharedAgent) podSpec(affinitySelector *metav1.LabelSelector) v1.PodSpec {
var limit v1.ResourceList
return v1.PodSpec{
Affinity: &v1.Affinity{
PodAntiAffinity: &v1.PodAntiAffinity{
@@ -132,13 +139,16 @@ func (s *SharedAgent) podSpec(image, name string, affinitySelector *metav1.Label
},
Containers: []v1.Container{
{
Name: name,
Image: image,
Name: s.Name(),
Image: s.sharedAgentImage,
ImagePullPolicy: v1.PullAlways,
Resources: v1.ResourceRequirements{
Limits: limit,
},
Args: args,
Args: []string{
"--config",
sharedKubeletConfigPath,
},
VolumeMounts: []v1.VolumeMount{
{
Name: "config",
@@ -243,14 +253,14 @@ func (s *SharedAgent) role() *rbacv1.Role {
},
Rules: []rbacv1.PolicyRule{
{
Verbs: []string{"*"},
APIGroups: []string{""},
Resources: []string{"pods", "pods/log", "pods/exec", "secrets", "configmaps", "services"},
Verbs: []string{"*"},
},
{
Verbs: []string{"get", "watch", "list"},
APIGroups: []string{"k3k.io"},
Resources: []string{"clusters"},
Verbs: []string{"get", "watch", "list"},
},
},
}
@@ -282,7 +292,7 @@ func (s *SharedAgent) roleBinding() *rbacv1.RoleBinding {
}
func (s *SharedAgent) Name() string {
return controller.SafeConcatNameWithPrefix(s.cluster.Name, sharedNodeAgentName)
return controller.SafeConcatNameWithPrefix(s.cluster.Name, SharedNodeAgentName)
}
func (s *SharedAgent) DNSName() string {
pkg/controller/cluster/cluster.go
+63 -5
View File
@@ -8,19 +8,21 @@ import (
"time"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
"github.com/rancher/k3k/pkg/controller/cluster/server"
"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
"github.com/rancher/k3k/pkg/log"
"go.uber.org/zap"
v1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
ctrlruntimecontroller "sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -59,7 +61,7 @@ func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage string, logg
}
return ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.Cluster{}).
WithOptions(controller.Options{
WithOptions(ctrlruntimecontroller.Options{
MaxConcurrentReconciles: maxConcurrentReconciles,
}).
Complete(&reconciler)
@@ -100,6 +102,11 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
}
}
}
if err := c.unbindNodeProxyClusterRole(ctx, &cluster); err != nil {
return reconcile.Result{}, err
}
if controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) {
// remove finalizer from the cluster and update it.
controllerutil.RemoveFinalizer(&cluster, clusterFinalizerName)
@@ -130,9 +137,6 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
cluster.Status.Persistence.StorageRequestSize = defaultStoragePersistentSize
}
}
if err := c.Client.Update(ctx, cluster); err != nil {
return err
}
cluster.Status.ClusterCIDR = cluster.Spec.ClusterCIDR
if cluster.Status.ClusterCIDR == "" {
@@ -189,6 +193,10 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
}
}
if err := c.bindNodeProxyClusterRole(ctx, cluster); err != nil {
return err
}
return c.Client.Update(ctx, cluster)
}
@@ -279,6 +287,56 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluste
return nil
}
func (c *ClusterReconciler) bindNodeProxyClusterRole(ctx context.Context, cluster *v1alpha1.Cluster) error {
clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
if err := c.Client.Get(ctx, types.NamespacedName{Name: "k3k-node-proxy"}, clusterRoleBinding); err != nil {
return fmt.Errorf("failed to get or find k3k-node-proxy ClusterRoleBinding: %w", err)
}
subjectName := controller.SafeConcatNameWithPrefix(cluster.Name, agent.SharedNodeAgentName)
found := false
for _, subject := range clusterRoleBinding.Subjects {
if subject.Name == subjectName && subject.Namespace == cluster.Namespace {
found = true
}
}
if !found {
clusterRoleBinding.Subjects = append(clusterRoleBinding.Subjects, rbacv1.Subject{
Kind: "ServiceAccount",
Name: subjectName,
Namespace: cluster.Namespace,
})
}
return c.Client.Update(ctx, clusterRoleBinding)
}
func (c *ClusterReconciler) unbindNodeProxyClusterRole(ctx context.Context, cluster *v1alpha1.Cluster) error {
clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
if err := c.Client.Get(ctx, types.NamespacedName{Name: "k3k-node-proxy"}, clusterRoleBinding); err != nil {
return fmt.Errorf("failed to get or find k3k-node-proxy ClusterRoleBinding: %w", err)
}
subjectName := controller.SafeConcatNameWithPrefix(cluster.Name, agent.SharedNodeAgentName)
var cleanedSubjects []rbacv1.Subject
for _, subject := range clusterRoleBinding.Subjects {
if subject.Name != subjectName || subject.Namespace != cluster.Namespace {
cleanedSubjects = append(cleanedSubjects, subject)
}
}
// if no subject was removed, all good
if reflect.DeepEqual(clusterRoleBinding.Subjects, cleanedSubjects) {
return nil
}
clusterRoleBinding.Subjects = cleanedSubjects
return c.Client.Update(ctx, clusterRoleBinding)
}
func (c *ClusterReconciler) agent(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP, token string) error {
agent := agent.New(cluster, serviceIP, c.SharedAgentImage, token)
agentsConfig, err := agent.Config()