Add structured logging via zap (#133)

* Add structured logging properly

use a centralized logger wrapper to work with controller and virt-kubelet

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* Fix some log messages

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fixes

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

---------

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
This commit is contained in:
Hussein Galal
2024-10-22 01:04:21 +03:00
committed by GitHub
parent d19f0f9ca6
commit f04902f0a2
11 changed files with 187 additions and 137 deletions
+24 -31
View File
@@ -8,16 +8,16 @@ import (
"time"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
k3kcontroller "github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
"github.com/rancher/k3k/pkg/controller/cluster/server"
"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
"github.com/rancher/k3k/pkg/log"
"go.uber.org/zap"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/klog"
ctrl "sigs.k8s.io/controller-runtime"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -45,15 +45,17 @@ type ClusterReconciler struct {
Client ctrlruntimeclient.Client
Scheme *runtime.Scheme
SharedAgentImage string
logger *log.Logger
}
// Add adds a new controller to the manager
func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage string) error {
func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage string, logger *log.Logger) error {
// initialize a new Reconciler
reconciler := ClusterReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
SharedAgentImage: sharedAgentImage,
logger: logger.Named(clusterController),
}
return ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.Cluster{}).
@@ -64,36 +66,29 @@ func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage string) erro
}
func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
var (
cluster v1alpha1.Cluster
podList v1.PodList
)
log := c.logger.With("Cluster", req.NamespacedName)
if err := c.Client.Get(ctx, req.NamespacedName, &cluster); err != nil {
return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
}
if cluster.DeletionTimestamp.IsZero() {
if !controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) {
controllerutil.AddFinalizer(&cluster, clusterFinalizerName)
if err := c.Client.Update(ctx, &cluster); err != nil {
return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to add cluster finalizer", err)
return reconcile.Result{}, err
}
}
klog.Infof("enqueue cluster [%s]", cluster.Name)
if err := c.createCluster(ctx, &cluster); err != nil {
return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to create cluster", err)
}
return reconcile.Result{}, nil
log.Info("enqueue cluster")
return reconcile.Result{}, c.createCluster(ctx, &cluster, log)
}
// remove finalizer from the server pods and update them.
matchingLabels := ctrlruntimeclient.MatchingLabels(map[string]string{"role": "server"})
listOpts := &ctrlruntimeclient.ListOptions{Namespace: cluster.Namespace}
matchingLabels.ApplyToList(listOpts)
if err := c.Client.List(ctx, &podList, listOpts); err != nil {
return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
}
@@ -101,26 +96,24 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
if controllerutil.ContainsFinalizer(&pod, etcdPodFinalizerName) {
controllerutil.RemoveFinalizer(&pod, etcdPodFinalizerName)
if err := c.Client.Update(ctx, &pod); err != nil {
return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to remove etcd finalizer", err)
return reconcile.Result{}, err
}
}
}
if controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) {
// remove finalizer from the cluster and update it.
controllerutil.RemoveFinalizer(&cluster, clusterFinalizerName)
if err := c.Client.Update(ctx, &cluster); err != nil {
return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to remove cluster finalizer", err)
return reconcile.Result{}, err
}
}
klog.Infof("deleting cluster [%s]", cluster.Name)
log.Info("deleting cluster")
return reconcile.Result{}, nil
}
func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1.Cluster) error {
func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1.Cluster, log *zap.SugaredLogger) error {
if err := c.validate(cluster); err != nil {
klog.Errorf("invalid change: %v", err)
log.Errorw("invalid change", zap.Error(err))
return nil
}
s := server.New(cluster, c.Client)
@@ -133,7 +126,7 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
}
}
if err := c.Client.Update(ctx, cluster); err != nil {
return k3kcontroller.LogAndReturnErr("failed to update cluster with persistence type", err)
return err
}
cluster.Status.ClusterCIDR = cluster.Spec.ClusterCIDR
@@ -146,35 +139,35 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
cluster.Status.ServiceCIDR = defaultClusterServiceCIDR
}
klog.Infof("creating cluster service")
log.Info("creating cluster service")
serviceIP, err := c.createClusterService(ctx, cluster, s)
if err != nil {
return k3kcontroller.LogAndReturnErr("failed to create cluster service", err)
return err
}
if err := c.createClusterConfigs(ctx, cluster, s, serviceIP); err != nil {
return k3kcontroller.LogAndReturnErr("failed to create cluster configs", err)
return err
}
// creating statefulsets in case the user chose a persistence type other than ephermal
if err := c.server(ctx, cluster, s); err != nil {
return k3kcontroller.LogAndReturnErr("failed to create servers", err)
return err
}
if err := c.agent(ctx, cluster, serviceIP); err != nil {
return k3kcontroller.LogAndReturnErr("failed to create agents", err)
return err
}
if cluster.Spec.Expose != nil {
if cluster.Spec.Expose.Ingress != nil {
serverIngress, err := s.Ingress(ctx, c.Client)
if err != nil {
return k3kcontroller.LogAndReturnErr("failed to create ingress object", err)
return err
}
if err := c.Client.Create(ctx, serverIngress); err != nil {
if !apierrors.IsAlreadyExists(err) {
return k3kcontroller.LogAndReturnErr("failed to create server ingress", err)
return err
}
}
}
@@ -182,12 +175,12 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
bootstrapSecret, err := bootstrap.Generate(ctx, cluster, serviceIP)
if err != nil {
return k3kcontroller.LogAndReturnErr("failed to generate new kubeconfig", err)
return err
}
if err := c.Client.Create(ctx, bootstrapSecret); err != nil {
if !apierrors.IsAlreadyExists(err) {
return k3kcontroller.LogAndReturnErr("failed to create kubeconfig secret", err)
return err
}
}
+19 -15
View File
@@ -15,16 +15,16 @@ import (
"github.com/rancher/k3k/pkg/controller/cluster/server"
"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
"github.com/rancher/k3k/pkg/controller/kubeconfig"
"github.com/sirupsen/logrus"
"github.com/rancher/k3k/pkg/log"
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
"k8s.io/klog"
ctrl "sigs.k8s.io/controller-runtime"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -41,14 +41,16 @@ const (
type PodReconciler struct {
Client ctrlruntimeclient.Client
Scheme *runtime.Scheme
logger *log.Logger
}
// Add adds a new controller to the manager
func AddPodController(ctx context.Context, mgr manager.Manager) error {
func AddPodController(ctx context.Context, mgr manager.Manager, logger *log.Logger) error {
// initialize a new Reconciler
reconciler := PodReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
logger: logger.Named(podController),
}
return ctrl.NewControllerManagedBy(mgr).
@@ -61,9 +63,11 @@ func AddPodController(ctx context.Context, mgr manager.Manager) error {
}
func (p *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
log := p.logger.With("Pod", req.NamespacedName)
s := strings.Split(req.Name, "-")
if len(s) < 1 {
return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to get cluster namespace", nil)
return reconcile.Result{}, nil
}
if s[0] != "k3k" {
return reconcile.Result{}, nil
@@ -84,15 +88,15 @@ func (p *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r
return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
}
for _, pod := range podList.Items {
klog.Infof("Handle etcd server pod [%s/%s]", pod.Namespace, pod.Name)
if err := p.handleServerPod(ctx, cluster, &pod); err != nil {
return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to handle etcd pod", err)
log.Info("Handle etcd server pod")
if err := p.handleServerPod(ctx, cluster, &pod, log); err != nil {
return reconcile.Result{}, err
}
}
return reconcile.Result{}, nil
}
func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cluster, pod *v1.Pod) error {
func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cluster, pod *v1.Pod, log *zap.SugaredLogger) error {
if _, ok := pod.Labels["role"]; ok {
if pod.Labels["role"] != "server" {
return nil
@@ -112,7 +116,7 @@ func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cl
}
return nil
}
tlsConfig, err := p.getETCDTLS(&cluster)
tlsConfig, err := p.getETCDTLS(&cluster, log)
if err != nil {
return err
}
@@ -127,7 +131,7 @@ func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cl
return err
}
if err := removePeer(ctx, client, pod.Name, pod.Status.PodIP); err != nil {
if err := removePeer(ctx, client, pod.Name, pod.Status.PodIP, log); err != nil {
return err
}
// remove our finalizer from the list and update it.
@@ -146,10 +150,10 @@ func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cl
return nil
}
func (p *PodReconciler) getETCDTLS(cluster *v1alpha1.Cluster) (*tls.Config, error) {
klog.Infof("generating etcd TLS client certificate for cluster [%s]", cluster.Name)
func (p *PodReconciler) getETCDTLS(cluster *v1alpha1.Cluster, log *zap.SugaredLogger) (*tls.Config, error) {
log.Infow("generating etcd TLS client certificate", "Cluster", cluster.Name, "Namespace", cluster.Namespace)
token := cluster.Spec.Token
endpoint := fmt.Sprintf("%s.%s", server.ServiceName(cluster.Name), cluster.Namespace)
endpoint := server.ServiceName(cluster.Name) + "." + cluster.Namespace
var b *bootstrap.ControlRuntimeBootstrap
if err := retry.OnError(k3kcontroller.Backoff, func(err error) bool {
return true
@@ -184,7 +188,7 @@ func (p *PodReconciler) getETCDTLS(cluster *v1alpha1.Cluster) (*tls.Config, erro
}
// removePeer removes a peer from the cluster. The peer name and IP address must both match.
func removePeer(ctx context.Context, client *clientv3.Client, name, address string) error {
func removePeer(ctx context.Context, client *clientv3.Client, name, address string, log *zap.SugaredLogger) error {
ctx, cancel := context.WithTimeout(ctx, memberRemovalTimeout)
defer cancel()
members, err := client.MemberList(ctx)
@@ -202,7 +206,7 @@ func removePeer(ctx context.Context, client *clientv3.Client, name, address stri
return err
}
if u.Hostname() == address {
logrus.Infof("Removing name=%s id=%d address=%s from etcd", member.Name, member.ID, address)
log.Infow("Removing member from etcd", "name", member.Name, "id", member.ID, "address", address)
_, err := client.MemberRemove(ctx, member.ID)
if errors.Is(err, rpctypes.ErrGRPCMemberNotFound) {
return nil
+10 -6
View File
@@ -2,10 +2,10 @@ package clusterset
import (
"context"
"fmt"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
k3kcontroller "github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/log"
v1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -29,15 +29,17 @@ type ClusterSetReconciler struct {
Client ctrlruntimeclient.Client
Scheme *runtime.Scheme
ClusterCIDR string
logger *log.Logger
}
// Add adds a new controller to the manager
func Add(ctx context.Context, mgr manager.Manager, clusterCIDR string) error {
func Add(ctx context.Context, mgr manager.Manager, clusterCIDR string, logger *log.Logger) error {
// initialize a new Reconciler
reconciler := ClusterSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ClusterCIDR: clusterCIDR,
logger: logger.Named(clusterSetController),
}
return ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.ClusterSet{}).
@@ -50,23 +52,25 @@ func Add(ctx context.Context, mgr manager.Manager, clusterCIDR string) error {
func (c *ClusterSetReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
var (
clusterSet v1alpha1.ClusterSet
log = c.logger.With("ClusterSet", req.NamespacedName)
)
if err := c.Client.Get(ctx, types.NamespacedName{Name: req.Name, Namespace: req.Namespace}, &clusterSet); err != nil {
return reconcile.Result{}, fmt.Errorf("unable to get the clusterset: %w", err)
return reconcile.Result{}, err
}
if !clusterSet.Spec.DisableNetworkPolicy {
log.Info("Creating NetworkPolicy")
setNetworkPolicy, err := netpol(ctx, c.ClusterCIDR, &clusterSet, c.Client)
if err != nil {
return reconcile.Result{}, fmt.Errorf("unable to make a networkpolicy for cluster set: %w", err)
return reconcile.Result{}, err
}
if err := c.Client.Create(ctx, setNetworkPolicy); err != nil {
if apierrors.IsAlreadyExists(err) {
if err := c.Client.Update(ctx, setNetworkPolicy); err != nil {
return reconcile.Result{}, fmt.Errorf("unable to update networkpolicy for clusterset: %w", err)
return reconcile.Result{}, err
}
}
return reconcile.Result{}, fmt.Errorf("unable to create networkpolicy for clusterset: %w", err)
return reconcile.Result{}, err
}
}
// TODO: Add resource quota for clustersets
+11 -5
View File
@@ -4,7 +4,8 @@ import (
"context"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
k3kcontroller "github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/log"
"go.uber.org/zap"
v1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -23,14 +24,16 @@ type NodeReconciler struct {
Client ctrlruntimeclient.Client
Scheme *runtime.Scheme
ClusterCIDR string
logger *log.Logger
}
// AddNodeController adds a new controller to the manager
func AddNodeController(ctx context.Context, mgr manager.Manager) error {
func AddNodeController(ctx context.Context, mgr manager.Manager, logger *log.Logger) error {
// initialize a new Reconciler
reconciler := NodeReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
logger: logger.Named(nodeController),
}
return ctrl.NewControllerManagedBy(mgr).
@@ -43,33 +46,36 @@ func AddNodeController(ctx context.Context, mgr manager.Manager) error {
}
func (n *NodeReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
log := n.logger.With("Node", req.NamespacedName)
var clusterSetList v1alpha1.ClusterSetList
if err := n.Client.List(ctx, &clusterSetList); err != nil {
return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to list clusterSets", err)
return reconcile.Result{}, err
}
if len(clusterSetList.Items) <= 0 {
return reconcile.Result{}, nil
}
if err := n.ensureNetworkPolicies(ctx, clusterSetList); err != nil {
if err := n.ensureNetworkPolicies(ctx, clusterSetList, log); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
}
func (n *NodeReconciler) ensureNetworkPolicies(ctx context.Context, clusterSetList v1alpha1.ClusterSetList) error {
func (n *NodeReconciler) ensureNetworkPolicies(ctx context.Context, clusterSetList v1alpha1.ClusterSetList, log *zap.SugaredLogger) error {
var setNetworkPolicy *networkingv1.NetworkPolicy
for _, cs := range clusterSetList.Items {
if cs.Spec.DisableNetworkPolicy {
continue
}
var err error
log.Infow("Updating NetworkPolicy for ClusterSet", "name", cs.Name, "namespace", cs.Namespace)
setNetworkPolicy, err = netpol(ctx, "", &cs, n.Client)
if err != nil {
return err
}
log.Debugw("New NetworkPolicy for clusterset", "name", cs.Name, "namespace", cs.Namespace)
if err := n.Client.Update(ctx, setNetworkPolicy); err != nil {
return err
}
-7
View File
@@ -10,7 +10,6 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -31,12 +30,6 @@ var Backoff = wait.Backoff{
func K3SImage(cluster *v1alpha1.Cluster) string {
return k3SImageName + ":" + cluster.Spec.Version
}
func LogAndReturnErr(errString string, err error) error {
klog.Errorf("%s: %v", errString, err)
return err
}
func nodeAddress(node *v1.Node) string {
var externalIP string
var internalIP string