feat: add log coupling for hostsensorutils

Signed-off-by: Alessio Greggi <ale_grey_91@hotmail.it>

core/pkg/hostsensorutils/hostsensordeploy.go

@@ -85,6 +85,10 @@ func (hsh *HostSensorHandler) Init(ctx context.Context) error {
 	logger.L().Info("Installing host scanner")
 	logger.L().Debug("The host scanner is a DaemonSet that runs on each node in the cluster. The DaemonSet will be running in it's own Namespace and will be deleted once the scan is completed. If you do not wish to install the host scanner, please run the scan without the --enable-host-scan flag.")
 
+	// log is used to avoid log duplication
+	// coming from the different host-scanner instances
+	log := NewLogCoupling()
+
 	cautils.StartSpinner()
 	defer cautils.StopSpinner()
 
@@ -92,9 +96,9 @@ func (hsh *HostSensorHandler) Init(ctx context.Context) error {
 		return fmt.Errorf("failed to apply host scanner YAML, reason: %v", err)
 	}
 
-	hsh.populatePodNamesToNodeNames(ctx)
+	hsh.populatePodNamesToNodeNames(ctx, log)
 	if err := hsh.checkPodForEachNode(); err != nil {
-		logger.L().Ctx(ctx).Warning("failed to validate host-scanner pods status", helpers.Error(err))
+		logger.L().Ctx(ctx).Warning(failedToValidateHostSensorPodStatus, helpers.Error(err))
 	}
 
 	return nil
@@ -157,7 +161,7 @@ func (hsh *HostSensorHandler) applyYAML(ctx context.Context) error {
 			containers, err := w.GetContainers()
 			if err != nil {
 				if erra := hsh.tearDownNamespace(namespaceName); erra != nil {
-					logger.L().Ctx(ctx).Warning("failed to tear down Namespace", helpers.Error(erra))
+					logger.L().Ctx(ctx).Warning(failedToTeardownNamespace, helpers.Error(erra))
 				}
 				return fmt.Errorf("container not found in DaemonSet: %v", err)
 			}
@@ -181,7 +185,7 @@ func (hsh *HostSensorHandler) applyYAML(ctx context.Context) error {
 		}
 		if e != nil {
 			if erra := hsh.tearDownNamespace(namespaceName); erra != nil {
-				logger.L().Ctx(ctx).Warning("failed to tear down Namespace", helpers.Error(erra))
+				logger.L().Ctx(ctx).Warning(failedToTeardownNamespace, helpers.Error(erra))
 			}
 			return fmt.Errorf("failed to create/update YAML, reason: %v", e)
 		}
@@ -191,14 +195,14 @@ func (hsh *HostSensorHandler) applyYAML(ctx context.Context) error {
 			b, err := json.Marshal(newWorkload.GetObject())
 			if err != nil {
 				if erra := hsh.tearDownNamespace(namespaceName); erra != nil {
-					logger.L().Ctx(ctx).Warning("failed to tear down Namespace", helpers.Error(erra))
+					logger.L().Ctx(ctx).Warning(failedToTeardownNamespace, helpers.Error(erra))
 				}
 				return fmt.Errorf("failed to Marshal YAML of DaemonSet, reason: %v", err)
 			}
 			var ds appsv1.DaemonSet
 			if err := json.Unmarshal(b, &ds); err != nil {
 				if erra := hsh.tearDownNamespace(namespaceName); erra != nil {
-					logger.L().Ctx(ctx).Warning("failed to tear down Namespace", helpers.Error(erra))
+					logger.L().Ctx(ctx).Warning(failedToTeardownNamespace, helpers.Error(erra))
 				}
 				return fmt.Errorf("failed to Unmarshal YAML of DaemonSet, reason: %v", err)
 			}
@@ -239,7 +243,7 @@ func (hsh *HostSensorHandler) checkPodForEachNode() error {
 }
 
 // initiating routine to keep pod list updated
-func (hsh *HostSensorHandler) populatePodNamesToNodeNames(ctx context.Context) {
+func (hsh *HostSensorHandler) populatePodNamesToNodeNames(ctx context.Context, log *LogsMap) {
 	go func() {
 		var watchRes watch.Interface
 		var err error
@@ -248,7 +252,7 @@ func (hsh *HostSensorHandler) populatePodNamesToNodeNames(ctx context.Context) {
 			LabelSelector: fmt.Sprintf("name=%s", hsh.daemonSet.Spec.Template.Labels["name"]),
 		})
 		if err != nil {
-			logger.L().Ctx(ctx).Warning("failed to watch over DaemonSet pods - are we missing watch pods permissions?", helpers.Error(err))
+			logger.L().Ctx(ctx).Warning(failedToWatchOverDaemonSetPods, helpers.Error(err))
 		}
 		if watchRes == nil {
 			logger.L().Ctx(ctx).Error("failed to watch over DaemonSet pods, will not be able to get host-scanner data")
@@ -260,12 +264,12 @@ func (hsh *HostSensorHandler) populatePodNamesToNodeNames(ctx context.Context) {
 			if !ok {
 				continue
 			}
-			go hsh.updatePodInListAtomic(ctx, eve.Type, pod)
+			go hsh.updatePodInListAtomic(ctx, eve.Type, pod, log)
 		}
 	}()
 }
 
-func (hsh *HostSensorHandler) updatePodInListAtomic(ctx context.Context, eventType watch.EventType, podObj *corev1.Pod) {
+func (hsh *HostSensorHandler) updatePodInListAtomic(ctx context.Context, eventType watch.EventType, podObj *corev1.Pod, log *LogsMap) {
 	hsh.podListLock.Lock()
 	defer hsh.podListLock.Unlock()
 
@@ -286,10 +290,11 @@ func (hsh *HostSensorHandler) updatePodInListAtomic(ctx context.Context, eventTy
 					len(podObj.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields[0].Values) > 0 {
 					nodeName = podObj.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields[0].Values[0]
 				}
-				logger.L().Ctx(ctx).Warning("One host-scanner pod is unable to schedule on node. We will fail to collect the data from this node",
-					helpers.String("message", podObj.Status.Conditions[0].Message),
-					helpers.String("nodeName", nodeName),
-					helpers.String("podName", podObj.ObjectMeta.Name))
+				if !log.isDuplicated(oneHostSensorPodIsUnabledToSchedule) {
+					logger.L().Ctx(ctx).Warning(oneHostSensorPodIsUnabledToSchedule,
+						helpers.String("message", podObj.Status.Conditions[0].Message))
+					log.update(oneHostSensorPodIsUnabledToSchedule)
+				}
 				if nodeName != "" {
 					hsh.hostSensorUnscheduledPodNames[podObj.ObjectMeta.Name] = nodeName
 				}

core/pkg/hostsensorutils/hostsensorgetfrompod.go

@@ -84,12 +84,17 @@ func (hsh *HostSensorHandler) sendAllPodsHTTPGETRequest(ctx context.Context, pat
 	podList := hsh.getPodList()
 	res := make([]hostsensor.HostSensorDataEnvelope, 0, len(podList))
 	var wg sync.WaitGroup
+
 	// initialization of the channels
 	hsh.workerPool.init(len(podList))
 
+	// log is used to avoid log duplication
+	// coming from the different host-scanner instances
+	log := NewLogCoupling()
+
 	hsh.workerPool.hostSensorApplyJobs(podList, path, requestKind)
 	hsh.workerPool.hostSensorGetResults(&res)
-	hsh.workerPool.createWorkerPool(ctx, hsh, &wg)
+	hsh.workerPool.createWorkerPool(ctx, hsh, &wg, log)
 	hsh.workerPool.waitForDone(&wg)
 
 	return res, nil

core/pkg/hostsensorutils/hostsensorworkerpool.go

@@ -43,22 +43,23 @@ func (wp *workerPool) init(noOfPods ...int) {
 }
 
 // The worker takes a job out of the chan, executes the request, and pushes the result to the results chan
-func (wp *workerPool) hostSensorWorker(ctx context.Context, hsh *HostSensorHandler, wg *sync.WaitGroup) {
+func (wp *workerPool) hostSensorWorker(ctx context.Context, hsh *HostSensorHandler, wg *sync.WaitGroup, log *LogsMap) {
 	defer wg.Done()
 	for job := range wp.jobs {
 		hostSensorDataEnvelope, err := hsh.getResourcesFromPod(job.podName, job.nodeName, job.requestKind, job.path)
-		if err != nil {
-			logger.L().Ctx(ctx).Warning("failed to get data", helpers.String("path", job.path), helpers.String("podName", job.podName), helpers.Error(err))
+		if err != nil && !log.isDuplicated(failedToGetData) {
+			logger.L().Ctx(ctx).Warning(failedToGetData, helpers.String("path", job.path), helpers.Error(err))
+			log.update(failedToGetData)
 			continue
 		}
 		wp.results <- hostSensorDataEnvelope
 	}
 }
 
-func (wp *workerPool) createWorkerPool(ctx context.Context, hsh *HostSensorHandler, wg *sync.WaitGroup) {
+func (wp *workerPool) createWorkerPool(ctx context.Context, hsh *HostSensorHandler, wg *sync.WaitGroup, log *LogsMap) {
 	for i := 0; i < noOfWorkers; i++ {
 		wg.Add(1)
-		go wp.hostSensorWorker(ctx, hsh, wg)
+		go wp.hostSensorWorker(ctx, hsh, wg, log)
 	}
 }
 

core/pkg/hostsensorutils/log_coupling.go

@@ -0,0 +1,51 @@
+package hostsensorutils
+
+import "sync"
+
+type LogsMap struct {
+	// use sync.Mutex to avoid read/write
+	// access issues in multi-thread environments.
+	sync.Mutex
+	usedLogs map[string]int
+}
+
+// NewLogCoupling return an empty LogsMap struct ready to be used.
+func NewLogCoupling() *LogsMap {
+	return &LogsMap{
+		usedLogs: make(map[string]int),
+	}
+}
+
+// update add the logContent to the internal map
+// and set the occurrencty to 1 (if it has never been used before),
+// increment its values otherwise.
+func (lm *LogsMap) update(logContent string) {
+	lm.Lock()
+	_, ok := lm.usedLogs[logContent]
+	if !ok {
+		lm.usedLogs[logContent] = 1
+	} else {
+		lm.usedLogs[logContent]++
+	}
+	lm.Unlock()
+}
+
+// isDuplicated check if logContent is already present inside the internal map.
+// Return true in case logContent already exists, false otherwise.
+func (lm *LogsMap) isDuplicated(logContent string) bool {
+	lm.Lock()
+	_, ok := lm.usedLogs[logContent]
+	lm.Unlock()
+	return ok
+}
+
+// GgtOccurrence retrieve the number of occurrences logContent has been used.
+func (lm *LogsMap) getOccurrence(logContent string) int {
+	lm.Lock()
+	occurrence, ok := lm.usedLogs[logContent]
+	lm.Unlock()
+	if !ok {
+		return 0
+	}
+	return occurrence
+}

core/pkg/hostsensorutils/log_coupling_test.go

@@ -0,0 +1,100 @@
+package hostsensorutils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLogsMap_Update(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name        string
+		logs        []string
+		expectedLog string
+		expected    int
+	}{
+		{
+			name: "test_1",
+			logs: []string{
+				"log_1",
+				"log_1",
+				"log_1",
+			},
+			expectedLog: "log_1",
+			expected:    3,
+		},
+		{
+			name:        "test_2",
+			logs:        []string{},
+			expectedLog: "log_2",
+			expected:    0,
+		},
+		{
+			name: "test_3",
+			logs: []string{
+				"log_3",
+			},
+			expectedLog: "log_3",
+			expected:    1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			lm := NewLogCoupling()
+			for _, log := range tt.logs {
+				lm.update(log)
+			}
+			if !assert.Equal(t, lm.getOccurrence(tt.expectedLog), tt.expected) {
+				t.Log("log occurrences are different")
+			}
+		})
+	}
+}
+
+func TestLogsMap_IsDuplicated(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name        string
+		logs        []string
+		expectedLog string
+		expected    bool
+	}{
+		{
+			name: "test_1",
+			logs: []string{
+				"log_1",
+				"log_1",
+				"log_1",
+			},
+			expectedLog: "log_1",
+			expected:    true,
+		},
+		{
+			name: "test_2",
+			logs: []string{
+				"log_1",
+				"log_1",
+			},
+			expectedLog: "log_2",
+			expected:    false,
+		},
+		{
+			name:        "test_3",
+			logs:        []string{},
+			expectedLog: "log_3",
+			expected:    false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			lm := NewLogCoupling()
+			for _, log := range tt.logs {
+				lm.update(log)
+			}
+			if !assert.Equal(t, lm.isDuplicated(tt.expectedLog), tt.expected) {
+				t.Log("duplication value differ from expected")
+			}
+		})
+	}
+}

core/pkg/hostsensorutils/logging_messages.go

@@ -0,0 +1,10 @@
+package hostsensorutils
+
+// messages used for warnings
+var (
+	failedToGetData                     = "failed to get data"
+	failedToTeardownNamespace           = "failed to teardown Namespace"
+	oneHostSensorPodIsUnabledToSchedule = "One host-sensor pod is unable to schedule on node. We will fail to collect the data from this node"
+	failedToWatchOverDaemonSetPods      = "failed to watch over DaemonSet pods"
+	failedToValidateHostSensorPodStatus = "failed to validate host-scanner pods status"
+)