From a3b928467ed8062d89ae9a00e54f4fa66dd381ab Mon Sep 17 00:00:00 2001 From: tashen Date: Wed, 10 Mar 2021 17:29:08 +0800 Subject: [PATCH] add loopbacktime to reduce time of journalctl call --- cmd/healthchecker/options/options.go | 3 +++ config/health-checker-kubelet.json | 1 + pkg/healthchecker/health_checker.go | 30 ++++++++++++++-------------- pkg/healthchecker/types/types.go | 1 + 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cmd/healthchecker/options/options.go b/cmd/healthchecker/options/options.go index 53940182..128c44d4 100644 --- a/cmd/healthchecker/options/options.go +++ b/cmd/healthchecker/options/options.go @@ -40,6 +40,7 @@ type HealthCheckerOptions struct { CriCtlPath string CriSocketPath string CoolDownTime time.Duration + LoopBackTime time.Duration HealthCheckTimeout time.Duration LogPatterns types.LogPatternFlag } @@ -63,6 +64,8 @@ func (hco *HealthCheckerOptions) AddFlags(fs *pflag.FlagSet) { "The path to the cri socket. Used with crictl to specify the socket path.") fs.DurationVar(&hco.CoolDownTime, "cooldown-time", types.DefaultCoolDownTime, "The duration to wait for the service to be up before attempting repair.") + fs.DurationVar(&hco.LoopBackTime, "loopback-time", types.DefaultLoopBackTime, + "The duration to loop back, if it is 0, health-check will check from start time.") fs.DurationVar(&hco.HealthCheckTimeout, "health-check-timeout", types.DefaultHealthCheckTimeout, "The time to wait before marking the component as unhealthy.") fs.Var(&hco.LogPatterns, "log-pattern", diff --git a/config/health-checker-kubelet.json b/config/health-checker-kubelet.json index 3d641b4f..994fc11d 100644 --- a/config/health-checker-kubelet.json +++ b/config/health-checker-kubelet.json @@ -25,6 +25,7 @@ "--component=kubelet", "--enable-repair=true", "--cooldown-time=1m", + "--loopback-time=0", "--health-check-timeout=10s" ], "timeout": "3m" diff --git a/pkg/healthchecker/health_checker.go b/pkg/healthchecker/health_checker.go index 77a96d29..9ab5c1b8 100644 --- a/pkg/healthchecker/health_checker.go +++ b/pkg/healthchecker/health_checker.go @@ -36,6 +36,7 @@ type healthChecker struct { crictlPath string healthCheckTimeout time.Duration coolDownTime time.Duration + loopBackTime time.Duration logPatternsToCheck map[string]int } @@ -48,6 +49,7 @@ func NewHealthChecker(hco *options.HealthCheckerOptions) (types.HealthChecker, e healthCheckTimeout: hco.HealthCheckTimeout, coolDownTime: hco.CoolDownTime, service: hco.Service, + loopBackTime: hco.LoopBackTime, logPatternsToCheck: hco.LogPatterns.GetLogPatternCountMap(), } hc.healthCheckFunc = getHealthCheckFunc(hco) @@ -59,11 +61,22 @@ func NewHealthChecker(hco *options.HealthCheckerOptions) (types.HealthChecker, e // CheckHealth checks for the health of the component and tries to repair if enabled. // Returns true if healthy, false otherwise. func (hc *healthChecker) CheckHealth() (bool, error) { + var logStartTime string healthy, err := hc.healthCheckFunc() if err != nil { return healthy, err } - logPatternHealthy, err := logPatternHealthCheck(hc.service, hc.logPatternsToCheck) + uptime, err := hc.uptimeFunc() + if err != nil { + glog.Warningf("Failed to get the uptime: %+v", err) + return true, err + } + if hc.loopBackTime > 0 && uptime > hc.loopBackTime { + logStartTime = time.Now().Add(-hc.loopBackTime).Format(types.LogParsingTimeLayout) + } else { + logStartTime = time.Now().Add(-uptime).Format(types.LogParsingTimeLayout) + } + logPatternHealthy, err := logPatternHealthCheck(hc.service, logStartTime, hc.logPatternsToCheck) if err != nil { return logPatternHealthy, err } @@ -74,10 +87,6 @@ func (hc *healthChecker) CheckHealth() (bool, error) { // Attempt repair based on flag. if hc.enableRepair { // repair if the service has been up for the cool down period. - uptime, err := hc.uptimeFunc() - if err != nil { - glog.Infof("error in getting uptime for %v: %v\n", hc.component, err) - } glog.Infof("%v is unhealthy, component uptime: %v\n", hc.component, uptime) if uptime > hc.coolDownTime { glog.Infof("%v cooldown period of %v exceeded, repairing", hc.component, hc.coolDownTime) @@ -89,19 +98,10 @@ func (hc *healthChecker) CheckHealth() (bool, error) { // logPatternHealthCheck checks for the provided logPattern occurrences in the service logs. // Returns true if the pattern is empty or does not exist logThresholdCount times since start of service, false otherwise. -func logPatternHealthCheck(service string, logPatternsToCheck map[string]int) (bool, error) { +func logPatternHealthCheck(service, logStartTime string, logPatternsToCheck map[string]int) (bool, error) { if len(logPatternsToCheck) == 0 { return true, nil } - uptimeFunc := getUptimeFunc(service) - uptime, err := uptimeFunc() - if err != nil { - return true, err - } - logStartTime := time.Now().Add(-uptime).Format(types.LogParsingTimeLayout) - if err != nil { - return true, err - } for pattern, count := range logPatternsToCheck { healthy, err := checkForPattern(service, logStartTime, pattern, count) if err != nil || !healthy { diff --git a/pkg/healthchecker/types/types.go b/pkg/healthchecker/types/types.go index a8585ff8..02523f6b 100644 --- a/pkg/healthchecker/types/types.go +++ b/pkg/healthchecker/types/types.go @@ -25,6 +25,7 @@ import ( ) const ( + DefaultLoopBackTime = 0 * time.Minute DefaultCoolDownTime = 2 * time.Minute DefaultHealthCheckTimeout = 10 * time.Second CmdTimeout = 10 * time.Second