From 3a9370e01b3cb49ac09e5e81f8e812718a527bc9 Mon Sep 17 00:00:00 2001
From: Archit Bansal <arba@google.com>
Date: Wed, 26 Aug 2020 23:17:04 -0700
Subject: [PATCH] Log custom plugin stderr only if the status is not ok.

Otherwise with plugins that run frequently and report ok status, the
logs are filled with unnecessary noise and significantly increases log
size.
---
 pkg/custompluginmonitor/plugin/plugin.go | 19 +++++++++++--------
 pkg/healthchecker/health_checker.go      | 11 ++++++-----
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/pkg/custompluginmonitor/plugin/plugin.go b/pkg/custompluginmonitor/plugin/plugin.go
index e5703a82..6738c2e8 100644
--- a/pkg/custompluginmonitor/plugin/plugin.go
+++ b/pkg/custompluginmonitor/plugin/plugin.go
@@ -89,7 +89,7 @@ func (p *Plugin) Run() {
 
 // run each rule in parallel and wait for them to complete
 func (p *Plugin) runRules() {
-	glog.Info("Start to run custom plugins")
+	glog.V(3).Info("Start to run custom plugins")
 
 	for _, rule := range p.config.Rules {
 		p.syncChan <- struct{}{}
@@ -120,7 +120,7 @@ func (p *Plugin) runRules() {
 	}
 
 	p.Wait()
-	glog.Info("Finish running custom plugins")
+	glog.V(3).Info("Finish running custom plugins")
 }
 
 // readFromReader reads the maxBytes from the reader and drains the rest.
@@ -203,12 +203,6 @@ func (p *Plugin) run(rule cpmtypes.CustomRule) (exitStatus cpmtypes.Status, outp
 		}
 	}
 
-	// log the stderr from the plugin
-	if len(stderr) != 0 {
-		glog.Infof("Start logs from plugin %q \n %s", rule.Path, string(stderr))
-		glog.Infof("End logs from plugin %q", rule.Path)
-	}
-
 	// trim suffix useless bytes
 	output = string(stdout)
 	output = strings.TrimSpace(output)
@@ -227,8 +221,10 @@ func (p *Plugin) run(rule cpmtypes.CustomRule) (exitStatus cpmtypes.Status, outp
 	case 0:
 		return cpmtypes.OK, output
 	case 1:
+		logPluginStderr(rule.Path, string(stderr))
 		return cpmtypes.NonOK, output
 	default:
+		logPluginStderr(rule.Path, string(stderr))
 		return cpmtypes.Unknown, output
 	}
 }
@@ -237,3 +233,10 @@ func (p *Plugin) Stop() {
 	p.tomb.Stop()
 	glog.Info("Stop plugin execution")
 }
+
+func logPluginStderr(path, logs string) {
+	if len(logs) != 0 {
+		glog.Infof("Start logs from plugin %q \n %s", path, string(logs))
+		glog.Infof("End logs from plugin %q", path)
+	}
+}
diff --git a/pkg/healthchecker/health_checker.go b/pkg/healthchecker/health_checker.go
index f4dfdf5c..e0eb448f 100644
--- a/pkg/healthchecker/health_checker.go
+++ b/pkg/healthchecker/health_checker.go
@@ -31,6 +31,7 @@ import (
 )
 
 type healthChecker struct {
+	component       string
 	enableRepair    bool
 	healthCheckFunc func() bool
 	// The repair is "best-effort" and ignores the error from the underlying actions.
@@ -45,6 +46,7 @@ type healthChecker struct {
 // NewHealthChecker returns a new health checker configured with the given options.
 func NewHealthChecker(hco *options.HealthCheckerOptions) (types.HealthChecker, error) {
 	hc := &healthChecker{
+		component:          hco.Component,
 		enableRepair:       hco.EnableRepair,
 		crictlPath:         hco.CriCtlPath,
 		healthCheckTimeout: hco.HealthCheckTimeout,
@@ -139,14 +141,14 @@ func (hc *healthChecker) CheckHealth() bool {
 	// The service is unhealthy.
 	// Attempt repair based on flag.
 	if hc.enableRepair {
-		glog.Infof("health-checker: component is unhealthy, proceeding to repair")
 		// repair if the service has been up for the cool down period.
 		uptime, err := hc.uptimeFunc()
 		if err != nil {
-			glog.Infof("health-checker: %v\n", err.Error())
+			glog.Infof("error in getting uptime for %v: %v\n", hc.component, err)
 		}
-		glog.Infof("health-checker: component uptime: %v\n", uptime)
+		glog.Infof("%v is unhealthy, component uptime: %v\n", hc.component, uptime)
 		if uptime > hc.coolDownTime {
+			glog.Infof("%v cooldown period of %v exceeded, repairing", hc.component, hc.coolDownTime)
 			hc.repairFunc()
 		}
 	}
@@ -159,10 +161,9 @@ func execCommand(timeout time.Duration, command string, args ...string) (string,
 	defer cancel()
 
 	cmd := exec.CommandContext(ctx, command, args...)
-	glog.Infof("health-checker: executing command : %v\n", cmd)
 	out, err := cmd.Output()
 	if err != nil {
-		glog.Infof("health-checker: command failed : %v, %v\n", err.Error(), out)
+		glog.Infof("command %v failed: %v, %v\n", cmd, err, out)
 		return "", err
 	}
 	return strings.TrimSuffix(string(out), "\n"), nil