From 29ff791f08a56ec0542a486456c39944778a6eff Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 3 Jun 2016 01:48:26 -0700 Subject: [PATCH] Hack for unsupported OS distros. --- pkg/kernelmonitor/kernel_log_watcher.go | 16 ++++++++++++++++ pkg/problemdetector/problem_detector.go | 6 +----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/pkg/kernelmonitor/kernel_log_watcher.go b/pkg/kernelmonitor/kernel_log_watcher.go index 5072b70c..c74f8c8e 100644 --- a/pkg/kernelmonitor/kernel_log_watcher.go +++ b/pkg/kernelmonitor/kernel_log_watcher.go @@ -71,6 +71,22 @@ func (k *kernelLogWatcher) Watch() (<-chan *types.KernelLog, error) { if k.cfg.KernelLogPath != "" { path = k.cfg.KernelLogPath } + // NOTE(random-liu): This is a hack. KernelMonitor doesn't support some OS distros e.g. GCI. Ideally, + // KernelMonitor should only run on nodes with supported OS distro. However, NodeProblemDetector is + // running as DaemonSet, it has to be deployed on each node (There is no node affinity support for + // DaemonSet now #22205). If some nodes have unsupported OS distro e.g. the OS distro of master node + // in gke/gce is GCI, KernelMonitor will keep throwing out error, and NodeProblemDetector will be + // restarted again and again. + // To avoid this, we decide to add this temporarily hack. When KernelMonitor can't find the kernel + // log file, it will print a log and then return nil channel and no error. Since nil channel will + // always be blocked, the NodeProblemDetector will block forever. + // TODO(random-liu): + // 1. Add journald supports to support GCI. + // 2. Schedule KernelMonitor only on supported node (with node label and selector) + if _, err := os.Stat(path); os.IsNotExist(err) { + glog.Infof("kernel log %q is not found, kernel monitor doesn't support the os distro", path) + return nil, nil + } start, err := k.getStartPoint(path) if err != nil { return nil, err diff --git a/pkg/problemdetector/problem_detector.go b/pkg/problemdetector/problem_detector.go index 04ff6e60..f1067d13 100644 --- a/pkg/problemdetector/problem_detector.go +++ b/pkg/problemdetector/problem_detector.go @@ -60,11 +60,7 @@ func (p *problemDetector) Run() error { glog.Info("Problem detector started") for { select { - case status, ok := <-ch: - if !ok { - glog.Errorf("Monitor stopped unexpectedly") - break - } + case status := <-ch: for _, event := range status.Events { p.client.Eventf(util.ConvertToAPIEventType(event.Severity), status.Source, event.Reason, event.Message) }