github/node-problem-detector
1
0
Fork 0
mirror of https://github.com/kubernetes/node-problem-detector.git synced 2026-08-19 20:26:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

add custom problem detector plugin

Browse Source
This commit is contained in:
Andy Xie
2017-11-22 10:14:09 +08:00
parent ffc790976b
commit 10dbfef1a8
31 changed files with 1058 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/systemlogmonitor/config.go
+21 -8
View File
@@ -17,8 +17,10 @@ limitations under the License.
package systemlogmonitor
import (
"regexp"
watchertypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types"
logtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
systemlogtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
"k8s.io/node-problem-detector/pkg/types"
)
@@ -33,15 +35,26 @@ type MonitorConfig struct {
// DefaultConditions are the default states of all the conditions log monitor should handle.
DefaultConditions []types.Condition `json:"conditions"`
// Rules are the rules log monitor will follow to parse the log file.
Rules []logtypes.Rule `json:"rules"`
Rules []systemlogtypes.Rule `json:"rules"`
}
// applyDefaultConfiguration applies default configurations.
func applyDefaultConfiguration(cfg *MonitorConfig) {
if cfg.BufferSize == 0 {
cfg.BufferSize = 10
// ApplyConfiguration applies default configurations.
func (mc *MonitorConfig) ApplyDefaultConfiguration() {
if mc.BufferSize == 0 {
mc.BufferSize = 10
}
if cfg.WatcherConfig.Lookback == "" {
cfg.WatcherConfig.Lookback = "0"
if mc.WatcherConfig.Lookback == "" {
mc.WatcherConfig.Lookback = "0"
}
}
// ValidateRules verifies whether the regular expressions in the rules are valid.
func (mc MonitorConfig) ValidateRules() error {
for _, rule := range mc.Rules {
_, err := regexp.Compile(rule.Pattern)
if err != nil {
return err
}
}
return nil
}
pkg/systemlogmonitor/log_monitor.go
+10 -31
View File
@@ -19,27 +19,18 @@ package systemlogmonitor
import (
"encoding/json"
"io/ioutil"
"regexp"
"time"
"k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers"
watchertypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types"
logtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
"k8s.io/node-problem-detector/pkg/systemlogmonitor/util"
systemlogtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
"k8s.io/node-problem-detector/pkg/types"
"github.com/golang/glog"
"k8s.io/node-problem-detector/pkg/util/tomb"
)
// LogMonitor monitors the log and reports node problem condition and event according to
// the rules.
type LogMonitor interface {
// Start starts the log monitor.
Start() (<-chan *types.Status, error)
// Stop stops the log monitor.
Stop()
}
type logMonitor struct {
watcher watchertypes.LogWatcher
buffer LogBuffer
@@ -47,13 +38,13 @@ type logMonitor struct {
conditions []types.Condition
logCh <-chan *logtypes.Log
output chan *types.Status
tomb *util.Tomb
tomb *tomb.Tomb
}
// NewLogMonitorOrDie create a new LogMonitor, panic if error occurs.
func NewLogMonitorOrDie(configPath string) LogMonitor {
func NewLogMonitorOrDie(configPath string) types.Monitor {
l := &logMonitor{
tomb: util.NewTomb(),
tomb: tomb.NewTomb(),
}
f, err := ioutil.ReadFile(configPath)
if err != nil {
@@ -64,10 +55,10 @@ func NewLogMonitorOrDie(configPath string) LogMonitor {
glog.Fatalf("Failed to unmarshal configuration file %q: %v", configPath, err)
}
// Apply default configurations
applyDefaultConfiguration(&l.config)
err = validateRules(l.config.Rules)
(&l.config).ApplyDefaultConfiguration()
err = l.config.ValidateRules()
if err != nil {
glog.Fatalf("Failed to validate matching rules %#v: %v", l.config.Rules, err)
glog.Fatalf("Failed to validate matching rules %+v: %v", l.config.Rules, err)
}
glog.Infof("Finish parsing log monitor config file: %+v", l.config)
l.watcher = logwatchers.GetLogWatcherOrDie(l.config.WatcherConfig)
@@ -126,12 +117,12 @@ func (l *logMonitor) parseLog(log *logtypes.Log) {
}
// generateStatus generates status from the logs.
func (l *logMonitor) generateStatus(logs []*logtypes.Log, rule logtypes.Rule) *types.Status {
func (l *logMonitor) generateStatus(logs []*logtypes.Log, rule systemlogtypes.Rule) *types.Status {
// We use the timestamp of the first log line as the timestamp of the status.
timestamp := logs[0].Timestamp
message := generateMessage(logs)
var events []types.Event
if rule.Type == logtypes.Temp {
if rule.Type == types.Temp {
// For temporary error only generate event
events = append(events, types.Event{
Severity: types.Warn,
@@ -144,7 +135,6 @@ func (l *logMonitor) generateStatus(logs []*logtypes.Log, rule logtypes.Rule) *t
for i := range l.conditions {
condition := &l.conditions[i]
if condition.Type == rule.Condition {
condition.Type = rule.Condition
// Update transition timestamp and message when the condition
// changes. Condition is considered to be changed only when
// status or reason changes.
@@ -189,17 +179,6 @@ func initialConditions(defaults []types.Condition) []types.Condition {
return conditions
}
// validateRules verifies whether the regular expressions in the rules are valid.
func validateRules(rules []logtypes.Rule) error {
for _, rule := range rules {
_, err := regexp.Compile(rule.Pattern)
if err != nil {
return err
}
}
return nil
}
func generateMessage(logs []*logtypes.Log) string {
messages := []string{}
for _, log := range logs {
pkg/systemlogmonitor/log_monitor_test.go
+3 -3
View File
@@ -67,7 +67,7 @@ func TestGenerateStatus(t *testing.T) {
// Do not need Pattern because we don't do pattern match in this test
{
rule: logtypes.Rule{
Type: logtypes.Perm,
Type: types.Perm,
Condition: testConditionA,
Reason: "test reason",
},
@@ -88,7 +88,7 @@ func TestGenerateStatus(t *testing.T) {
// Should not update transition time when status and reason are not changed.
{
rule: logtypes.Rule{
Type: logtypes.Perm,
Type: types.Perm,
Condition: testConditionA,
Reason: "initial reason",
},
@@ -107,7 +107,7 @@ func TestGenerateStatus(t *testing.T) {
},
{
rule: logtypes.Rule{
Type: logtypes.Temp,
Type: types.Temp,
Reason: "test reason",
},
expected: types.Status{
pkg/systemlogmonitor/logwatchers/filelog/log_watcher.go
+3 -3
View File
@@ -32,7 +32,7 @@ import (
"k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types"
logtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
"k8s.io/node-problem-detector/pkg/systemlogmonitor/util"
"k8s.io/node-problem-detector/pkg/util/tomb"
)
type filelogWatcher struct {
@@ -42,7 +42,7 @@ type filelogWatcher struct {
translator *translator
logCh chan *logtypes.Log
uptime time.Time
tomb *util.Tomb
tomb *tomb.Tomb
clock utilclock.Clock
}
@@ -57,7 +57,7 @@ func NewSyslogWatcherOrDie(cfg types.WatcherConfig) types.LogWatcher {
cfg: cfg,
translator: newTranslatorOrDie(cfg.PluginConfig),
uptime: time.Now().Add(time.Duration(-info.Uptime * int64(time.Second))),
tomb: util.NewTomb(),
tomb: tomb.NewTomb(),
// A capacity 1000 buffer should be enough
logCh: make(chan *logtypes.Log, 1000),
clock: utilclock.NewClock(),
pkg/systemlogmonitor/logwatchers/journald/log_watcher.go
+3 -3
View File
@@ -30,7 +30,7 @@ import (
"k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types"
logtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
"k8s.io/node-problem-detector/pkg/systemlogmonitor/util"
"k8s.io/node-problem-detector/pkg/util/tomb"
)
// Compiling go-systemd/sdjournald needs libsystemd-dev or libsystemd-journal-dev,
@@ -43,14 +43,14 @@ type journaldWatcher struct {
journal *sdjournal.Journal
cfg types.WatcherConfig
logCh chan *logtypes.Log
tomb *util.Tomb
tomb *tomb.Tomb
}
// NewJournaldWatcher is the create function of journald watcher.
func NewJournaldWatcher(cfg types.WatcherConfig) types.LogWatcher {
return &journaldWatcher{
cfg: cfg,
tomb: util.NewTomb(),
tomb: tomb.NewTomb(),
// A capacity 1000 buffer should be enough
logCh: make(chan *logtypes.Log, 1000),
}
pkg/systemlogmonitor/logwatchers/kmsg/log_watcher.go
+3 -3
View File
@@ -27,13 +27,13 @@ import (
"k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types"
logtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
"k8s.io/node-problem-detector/pkg/systemlogmonitor/util"
"k8s.io/node-problem-detector/pkg/util/tomb"
)
type kernelLogWatcher struct {
cfg types.WatcherConfig
logCh chan *logtypes.Log
tomb *util.Tomb
tomb *tomb.Tomb
kmsgParser kmsgparser.Parser
clock utilclock.Clock
@@ -43,7 +43,7 @@ type kernelLogWatcher struct {
func NewKmsgWatcher(cfg types.WatcherConfig) types.LogWatcher {
return &kernelLogWatcher{
cfg: cfg,
tomb: util.NewTomb(),
tomb: tomb.NewTomb(),
// Arbitrary capacity
logCh: make(chan *logtypes.Log, 100),
clock: utilclock.NewClock(),
pkg/systemlogmonitor/logwatchers/types/log_watcher.go
+1 -1
View File
@@ -31,7 +31,7 @@ type LogWatcher interface {
// WatcherConfig is the configuration of the log watcher.
type WatcherConfig struct {
// Plugin is the name of plugin which is currently used.
// Currently supported: filelog, journald.
// Currently supported: filelog, journald, kmsg.
Plugin string `json:"plugin, omitempty"`
// PluginConfig is a key/value configuration of a plugin. Valid configurations
// are defined in different log watcher plugin.
pkg/systemlogmonitor/types/types.go
+2 -11
View File
@@ -17,6 +17,7 @@ limitations under the License.
package types
import (
"k8s.io/node-problem-detector/pkg/types"
"time"
)
@@ -27,20 +28,10 @@ type Log struct {
Message string
}
// Type is the type of the problem.
type Type string
const (
// Temp means the problem is temporary, only need to report an event.
Temp Type = "temporary"
// Perm means the problem is permanent, need to change the node condition.
Perm Type = "permanent"
)
// Rule describes how log monitor should analyze the log.
type Rule struct {
// Type is the type of matched problem.
Type Type `json:"type"`
Type types.Type `json:"type"`
// Condition is the type of the condition the problem triggered. Notice that
// the Condition field should be set only when the problem is permanent, or
// else the field will be ignored.
pkg/systemlogmonitor/util/tomb.go
-47
View File
@@ -1,47 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
// Tomb is used to control the lifecycle of a goroutine.
type Tomb struct {
stop chan struct{}
done chan struct{}
}
// NewTomb creates a new tomb.
func NewTomb() *Tomb {
return &Tomb{
stop: make(chan struct{}),
done: make(chan struct{}),
}
}
// Stop is used to stop the goroutine outside.
func (t *Tomb) Stop() {
close(t.stop)
<-t.done
}
// Stopping is used by the goroutine to tell whether it should stop.
func (t *Tomb) Stopping() <-chan struct{} {
return t.stop
}
// Done is used by the goroutine to inform that it has stopped.
func (t *Tomb) Done() {
close(t.done)
}
pkg/systemlogmonitor/util/tomb_test.go
-39
View File
@@ -1,39 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"reflect"
"testing"
)
func TestTomb(t *testing.T) {
tomb := NewTomb()
workflow := []string{}
expected := []string{"stop", "stopping", "stopped"}
go func() {
defer tomb.Done()
<-tomb.Stopping()
workflow = append(workflow, "stopping")
}()
workflow = append(workflow, "stop")
tomb.Stop()
workflow = append(workflow, "stopped")
if !reflect.DeepEqual(workflow, expected) {
t.Errorf("expected workflow %v, got %v", expected, workflow)
}
}