mirror of
https://github.com/kubernetes/node-problem-detector.git
synced 2026-08-28 01:47:20 +00:00
Add first version of node-problem-detector
This commit is contained in:
Lantao Liu
@@ -0,0 +1,166 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kernelmonitor
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
|
||||
"k8s.io/node-problem-detector/pkg/kernelmonitor/translator"
|
||||
"k8s.io/node-problem-detector/pkg/kernelmonitor/types"
|
||||
"k8s.io/node-problem-detector/pkg/kernelmonitor/util"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"github.com/hpcloud/tail"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultKernelLogPath = "/var/log/kern.log"
|
||||
)
|
||||
|
||||
// WatcherConfig is the configuration of kernel log watcher.
|
||||
type WatcherConfig struct {
|
||||
KernelLogPath string `json:"logPath, omitempty"`
|
||||
}
|
||||
|
||||
// KernelLogWatcher watches and translates the kernel log. Once there is new log line,
|
||||
// it will translate and report the log.
|
||||
type KernelLogWatcher interface {
|
||||
// Watch starts the kernel log watcher and returns a watch channel.
|
||||
Watch() (<-chan *types.KernelLog, error)
|
||||
// Stop stops the kernel log watcher.
|
||||
Stop()
|
||||
}
|
||||
|
||||
type kernelLogWatcher struct {
|
||||
// trans is the translator translates the log into internal format.
|
||||
trans translator.Translator
|
||||
cfg WatcherConfig
|
||||
tl *tail.Tail
|
||||
logCh chan *types.KernelLog
|
||||
tomb *util.Tomb
|
||||
}
|
||||
|
||||
// NewKernelLogWatcher creates a new kernel log watcher.
|
||||
func NewKernelLogWatcher(cfg WatcherConfig) KernelLogWatcher {
|
||||
return &kernelLogWatcher{
|
||||
trans: translator.NewDefaultTranslator(),
|
||||
cfg: cfg,
|
||||
tomb: util.NewTomb(),
|
||||
// A capacity 1000 buffer should be enough
|
||||
logCh: make(chan *types.KernelLog, 1000),
|
||||
}
|
||||
}
|
||||
|
||||
func (k *kernelLogWatcher) Watch() (<-chan *types.KernelLog, error) {
|
||||
path := defaultKernelLogPath
|
||||
if k.cfg.KernelLogPath != "" {
|
||||
path = k.cfg.KernelLogPath
|
||||
}
|
||||
start, err := k.getStartPoint(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// TODO(random-liu): If the file gets recreated during this interval, the logic
|
||||
// will go wrong here.
|
||||
// TODO(random-liu): Rate limit tail file.
|
||||
// TODO(random-liu): Figure out what happens if log lines are removed.
|
||||
k.tl, err = tail.TailFile(path, tail.Config{
|
||||
Location: &tail.SeekInfo{
|
||||
Offset: start,
|
||||
Whence: os.SEEK_SET,
|
||||
},
|
||||
Poll: true,
|
||||
ReOpen: true,
|
||||
Follow: true,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
glog.Info("Start watching kernel log")
|
||||
go k.watchLoop()
|
||||
return k.logCh, nil
|
||||
}
|
||||
|
||||
func (k *kernelLogWatcher) Stop() {
|
||||
k.tomb.Stop()
|
||||
}
|
||||
|
||||
// watchLoop is the main watch loop of kernel log watcher.
|
||||
func (k *kernelLogWatcher) watchLoop() {
|
||||
defer func() {
|
||||
close(k.logCh)
|
||||
k.tomb.Done()
|
||||
}()
|
||||
for {
|
||||
select {
|
||||
case line := <-k.tl.Lines:
|
||||
// Notice that tail has trimmed '\n'
|
||||
if line.Err != nil {
|
||||
glog.Errorf("Tail error: %v", line.Err)
|
||||
continue
|
||||
}
|
||||
log, err := k.trans.Translate(line.Text)
|
||||
if err != nil {
|
||||
glog.Infof("Unable to parse line: %q, %v", line, err)
|
||||
continue
|
||||
}
|
||||
k.logCh <- log
|
||||
case <-k.tomb.Stopping():
|
||||
k.tl.Stop()
|
||||
glog.Infof("Stop watching kernel log")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getStartPoint parses the newest kernel log file and try to find the latest reboot point.
|
||||
// Currently we rely on the kernel log timestamp to find the reboot point. The basic idea
|
||||
// is straight forward: In the whole lifecycle of a node, the kernel log timestamp should
|
||||
// always increase, only when it is reboot, the timestamp will decrease. We just parse the
|
||||
// log and find the latest timestamp decreasing, then it should be the latest reboot point.
|
||||
// TODO(random-liu): A drawback is that if the node is started long time ago, we'll only get
|
||||
// logs in the newest kernel log file. We may want to improve this in the future.
|
||||
func (k *kernelLogWatcher) getStartPoint(path string) (int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
defer f.Close()
|
||||
start := int64(0)
|
||||
total := 0
|
||||
lastTimestamp := int64(0)
|
||||
reader := bufio.NewReader(f)
|
||||
done := false
|
||||
for !done {
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
done = true
|
||||
}
|
||||
total += len(line)
|
||||
log, err := k.trans.Translate(line)
|
||||
if err != nil {
|
||||
glog.Infof("unable to parse line: %q, %v", line, err)
|
||||
continue
|
||||
}
|
||||
if log.Timestamp < lastTimestamp {
|
||||
start = int64(total - len(line))
|
||||
}
|
||||
lastTimestamp = log.Timestamp
|
||||
}
|
||||
return start, nil
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kernelmonitor
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"k8s.io/node-problem-detector/pkg/kernelmonitor/types"
|
||||
)
|
||||
|
||||
func TestGetStartPoint(t *testing.T) {
|
||||
testCases := []struct {
|
||||
log string
|
||||
logs []types.KernelLog
|
||||
err bool
|
||||
}{
|
||||
{
|
||||
// The start point is at the head of the log file.
|
||||
log: `kernel: [1.000000] 1
|
||||
kernel: [2.000000] 2
|
||||
kernel: [3.000000] 3
|
||||
`,
|
||||
logs: []types.KernelLog{
|
||||
{
|
||||
Timestamp: 1000000,
|
||||
Message: "1",
|
||||
},
|
||||
{
|
||||
Timestamp: 2000000,
|
||||
Message: "2",
|
||||
},
|
||||
{
|
||||
Timestamp: 3000000,
|
||||
Message: "3",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// The start point is in the middle of the log file.
|
||||
log: `kernel: [3.000000] 3
|
||||
kernel: [1.000000] 1
|
||||
kernel: [2.000000] 2
|
||||
`,
|
||||
logs: []types.KernelLog{
|
||||
{
|
||||
Timestamp: 1000000,
|
||||
Message: "1",
|
||||
},
|
||||
{
|
||||
Timestamp: 2000000,
|
||||
Message: "2",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for c, test := range testCases {
|
||||
f, err := ioutil.TempFile("", "kernel_log_watcher_test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
}()
|
||||
_, err = f.Write([]byte(test.log))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := NewKernelLogWatcher(WatcherConfig{KernelLogPath: f.Name()})
|
||||
logCh, err := w.Watch()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer w.Stop()
|
||||
for _, expected := range test.logs {
|
||||
got := <-logCh
|
||||
if !reflect.DeepEqual(&expected, got) {
|
||||
t.Errorf("case %d: expect %+v, got %+v", c+1, expected, *got)
|
||||
}
|
||||
}
|
||||
// The log channel should have already been drained
|
||||
// There could stil be future messages sent into the channel, but the chance is really slim.
|
||||
timeout := time.After(100 * time.Millisecond)
|
||||
select {
|
||||
case log := <-logCh:
|
||||
t.Errorf("unexpected extra log: %+v", *log)
|
||||
case <-timeout:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kernelmonitor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"regexp"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
kerntypes "k8s.io/node-problem-detector/pkg/kernelmonitor/types"
|
||||
"k8s.io/node-problem-detector/pkg/kernelmonitor/util"
|
||||
"k8s.io/node-problem-detector/pkg/types"
|
||||
|
||||
"github.com/golang/glog"
|
||||
)
|
||||
|
||||
// May want to add more conditions if we need finer grained node conditions.
|
||||
// TODO(random-liu): Make the kernel condition to be a predefined list, and make it configurable
|
||||
// in rule.
|
||||
const (
|
||||
KernelDeadlockCondition = "KernelDeadlock"
|
||||
KernelMonitorSource = "kernel-monitor"
|
||||
)
|
||||
|
||||
// MonitorConfig is the configuration of kernel monitor.
|
||||
type MonitorConfig struct {
|
||||
// WatcherConfig is the configuration of kernel log watcher.
|
||||
WatcherConfig
|
||||
// BufferSize is the size (in lines) of the log buffer.
|
||||
BufferSize int `json:"bufferSize"`
|
||||
// Rules are the rules kernel monitor will follow to parse the log file.
|
||||
Rules []kerntypes.Rule `json:"rules"`
|
||||
}
|
||||
|
||||
// KernelMonitor monitors the kernel log and reports node problem condition and event according to
|
||||
// the rules.
|
||||
type KernelMonitor interface {
|
||||
// Start starts the kernel monitor.
|
||||
Start() (<-chan *types.Status, error)
|
||||
// Stop stops the kernel monitor.
|
||||
Stop()
|
||||
}
|
||||
|
||||
type kernelMonitor struct {
|
||||
watcher KernelLogWatcher
|
||||
buffer LogBuffer
|
||||
config MonitorConfig
|
||||
condition types.Condition
|
||||
uptime time.Time
|
||||
logCh <-chan *kerntypes.KernelLog
|
||||
output chan *types.Status
|
||||
tomb *util.Tomb
|
||||
}
|
||||
|
||||
// NewKernelMonitorOrDie create a new KernelMonitor, panic if error occurs.
|
||||
func NewKernelMonitorOrDie(configPath string) KernelMonitor {
|
||||
k := &kernelMonitor{
|
||||
condition: defaultCondition(),
|
||||
tomb: util.NewTomb(),
|
||||
}
|
||||
f, err := ioutil.ReadFile(configPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = json.Unmarshal(f, &k.config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = validateRules(k.config.Rules)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
glog.Infof("Finish parsing log file: %+v", k.config)
|
||||
var info syscall.Sysinfo_t
|
||||
err = syscall.Sysinfo(&info)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
k.uptime = time.Now().Add(time.Duration(-info.Uptime * int64(time.Second)))
|
||||
glog.Infof("Got system boot time: %v", k.uptime)
|
||||
k.watcher = NewKernelLogWatcher(k.config.WatcherConfig)
|
||||
k.buffer = NewLogBuffer(k.config.BufferSize)
|
||||
// A 1000 size channel should be big enough.
|
||||
k.output = make(chan *types.Status, 1000)
|
||||
return k
|
||||
}
|
||||
|
||||
func (k *kernelMonitor) Start() (<-chan *types.Status, error) {
|
||||
glog.Info("Start kernel monitor")
|
||||
var err error
|
||||
k.logCh, err = k.watcher.Watch()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
go k.monitorLoop()
|
||||
return k.output, nil
|
||||
}
|
||||
|
||||
func (k *kernelMonitor) Stop() {
|
||||
glog.Info("Stop kernel monitor")
|
||||
k.tomb.Stop()
|
||||
}
|
||||
|
||||
// monitorLoop is the main loop of kernel monitor.
|
||||
func (k *kernelMonitor) monitorLoop() {
|
||||
defer k.tomb.Done()
|
||||
k.output <- defaultStatus() // Update the default status
|
||||
for {
|
||||
select {
|
||||
case log := <-k.logCh:
|
||||
// Once there is new log, kernel monitor will push it into the log buffer and try
|
||||
// to match each rule. If any rule is matched, kernel monitor will report a status.
|
||||
k.buffer.Push(log)
|
||||
for _, rule := range k.config.Rules {
|
||||
matched := k.buffer.Match(rule.Pattern)
|
||||
if len(matched) == 0 {
|
||||
continue
|
||||
}
|
||||
status := k.generateStatus(matched, rule)
|
||||
glog.Infof("New status generated: %+v", status)
|
||||
k.output <- status
|
||||
}
|
||||
case <-k.tomb.Stopping():
|
||||
k.watcher.Stop()
|
||||
glog.Infof("Kernel monitor stopped")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// generateStatus generates status from the logs.
|
||||
func (k *kernelMonitor) generateStatus(logs []*kerntypes.KernelLog, rule kerntypes.Rule) *types.Status {
|
||||
// We use the timestamp of the first log line as the timestamp of the status.
|
||||
timestamp := k.generateTimestamp(logs[0].Timestamp)
|
||||
messages := []string{}
|
||||
for _, log := range logs {
|
||||
messages = append(messages, log.Message)
|
||||
}
|
||||
message := concatLogs(messages)
|
||||
var event *types.Event
|
||||
if rule.Type == kerntypes.Temp {
|
||||
// For temporary error only generate event
|
||||
event = &types.Event{
|
||||
Severity: types.Warn,
|
||||
Timestamp: timestamp,
|
||||
Reason: rule.Reason,
|
||||
Message: message,
|
||||
}
|
||||
} else {
|
||||
// For permanent error changes the condition
|
||||
k.condition.Type = KernelDeadlockCondition
|
||||
k.condition.Status = true
|
||||
k.condition.Transition = timestamp
|
||||
k.condition.Reason = rule.Reason
|
||||
k.condition.Message = message
|
||||
}
|
||||
return &types.Status{
|
||||
Source: KernelMonitorSource,
|
||||
Event: event,
|
||||
Condition: k.condition,
|
||||
}
|
||||
}
|
||||
|
||||
// generateTimestamp converts the kernel log time to real time.
|
||||
func (k *kernelMonitor) generateTimestamp(timestamp int64) time.Time {
|
||||
return k.uptime.Add(time.Duration(timestamp * int64(time.Microsecond)))
|
||||
}
|
||||
|
||||
// defaultStatus returns the default status with default condition.
|
||||
func defaultStatus() *types.Status {
|
||||
return &types.Status{
|
||||
Source: KernelMonitorSource,
|
||||
Condition: defaultCondition(),
|
||||
}
|
||||
}
|
||||
|
||||
func defaultCondition() types.Condition {
|
||||
return types.Condition{
|
||||
Type: KernelDeadlockCondition,
|
||||
Status: false,
|
||||
Transition: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// validateRules verifies whether the regular expressions in the rules are valid.
|
||||
func validateRules(rules []kerntypes.Rule) error {
|
||||
for _, rule := range rules {
|
||||
_, err := regexp.Compile(rule.Pattern)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kernelmonitor
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
kerntypes "k8s.io/node-problem-detector/pkg/kernelmonitor/types"
|
||||
"k8s.io/node-problem-detector/pkg/types"
|
||||
)
|
||||
|
||||
func TestGenerateStatus(t *testing.T) {
|
||||
uptime := time.Unix(1000, 0)
|
||||
initCondition := defaultCondition()
|
||||
logs := []*kerntypes.KernelLog{
|
||||
{
|
||||
Timestamp: 100000,
|
||||
Message: "test message 1",
|
||||
},
|
||||
{
|
||||
Timestamp: 200000,
|
||||
Message: "test message 2",
|
||||
},
|
||||
}
|
||||
for c, test := range []struct {
|
||||
rule kerntypes.Rule
|
||||
expected types.Status
|
||||
}{
|
||||
// Do not need Pattern because we don't do pattern match in this test
|
||||
{
|
||||
rule: kerntypes.Rule{
|
||||
Type: kerntypes.Perm,
|
||||
Reason: "test reason",
|
||||
},
|
||||
expected: types.Status{
|
||||
Source: KernelMonitorSource,
|
||||
Condition: types.Condition{
|
||||
Type: KernelDeadlockCondition,
|
||||
Status: true,
|
||||
Transition: time.Unix(1000, 100000*1000),
|
||||
Reason: "test reason",
|
||||
Message: "test message 1\ntest message 2",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
rule: kerntypes.Rule{
|
||||
Type: kerntypes.Temp,
|
||||
Reason: "test reason",
|
||||
},
|
||||
expected: types.Status{
|
||||
Source: KernelMonitorSource,
|
||||
Event: &types.Event{
|
||||
Severity: types.Warn,
|
||||
Timestamp: time.Unix(1000, 100000*1000),
|
||||
Reason: "test reason",
|
||||
Message: "test message 1\ntest message 2",
|
||||
},
|
||||
Condition: initCondition,
|
||||
},
|
||||
},
|
||||
} {
|
||||
k := &kernelMonitor{
|
||||
condition: initCondition,
|
||||
uptime: uptime,
|
||||
}
|
||||
got := k.generateStatus(logs, test.rule)
|
||||
if !reflect.DeepEqual(&test.expected, got) {
|
||||
t.Errorf("case %d: expected status %+v, got %+v", c+1, test.expected, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kernelmonitor
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"k8s.io/node-problem-detector/pkg/kernelmonitor/types"
|
||||
)
|
||||
|
||||
// LogBuffer buffers the logs and supports match in the log buffer with regular expression.
|
||||
type LogBuffer interface {
|
||||
// Push pushes log into the log buffer.
|
||||
Push(*types.KernelLog)
|
||||
// Match with regular expression in the log buffer.
|
||||
Match(string) []*types.KernelLog
|
||||
// String returns a concatenated string of the buffered logs.
|
||||
String() string
|
||||
}
|
||||
|
||||
type logBuffer struct {
|
||||
// buffer is a simple ring buffer.
|
||||
buffer []*types.KernelLog
|
||||
msg []string
|
||||
max int
|
||||
current int
|
||||
}
|
||||
|
||||
// NewLogBuffer creates log buffer with max line number limit. Because we only match logs
|
||||
// in the log buffer, the max buffer line number is also the max pattern line number we
|
||||
// support. Smaller buffer line number means less memory and cpu usage, but also means less
|
||||
// lines of patterns we support.
|
||||
func NewLogBuffer(maxLines int) *logBuffer {
|
||||
return &logBuffer{
|
||||
buffer: make([]*types.KernelLog, maxLines, maxLines),
|
||||
msg: make([]string, maxLines, maxLines),
|
||||
max: maxLines,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *logBuffer) Push(log *types.KernelLog) {
|
||||
b.buffer[b.current%b.max] = log
|
||||
b.msg[b.current%b.max] = log.Message
|
||||
b.current++
|
||||
}
|
||||
|
||||
// TODO(random-liu): Cache regexp if garbage collection becomes a problem someday.
|
||||
func (b *logBuffer) Match(expr string) []*types.KernelLog {
|
||||
// The expression should be checked outside, and it must match to the end.
|
||||
reg := regexp.MustCompile(expr + `\z`)
|
||||
log := b.String()
|
||||
loc := reg.FindStringIndex(log)
|
||||
if loc == nil {
|
||||
// No match
|
||||
return nil
|
||||
}
|
||||
// reverse index
|
||||
s := len(log) - loc[0] - 1
|
||||
total := 0
|
||||
matched := []*types.KernelLog{}
|
||||
for i := b.tail(); i >= b.current && b.buffer[i%b.max] != nil; i-- {
|
||||
matched = append(matched, b.buffer[i%b.max])
|
||||
total += len(b.msg[i%b.max]) + 1 // Add '\n'
|
||||
if total > s {
|
||||
break
|
||||
}
|
||||
}
|
||||
for i := 0; i < len(matched)/2; i++ {
|
||||
matched[i], matched[len(matched)-i-1] = matched[len(matched)-i-1], matched[i]
|
||||
}
|
||||
return matched
|
||||
}
|
||||
|
||||
func (b *logBuffer) String() string {
|
||||
logs := append(b.msg[b.current%b.max:], b.msg[:b.current%b.max]...)
|
||||
return concatLogs(logs)
|
||||
}
|
||||
|
||||
// tail returns current tail index.
|
||||
func (b *logBuffer) tail() int {
|
||||
return b.current + b.max - 1
|
||||
}
|
||||
|
||||
// concatLogs concatenates multiple lines of logs into one string.
|
||||
func concatLogs(logs []string) string {
|
||||
return strings.Join(logs, "\n")
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kernelmonitor
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"k8s.io/node-problem-detector/pkg/kernelmonitor/types"
|
||||
)
|
||||
|
||||
func TestPush(t *testing.T) {
|
||||
for c, test := range []struct {
|
||||
max int
|
||||
logs []string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
max: 1,
|
||||
logs: []string{"a", "b"},
|
||||
expected: "b",
|
||||
},
|
||||
{
|
||||
max: 2,
|
||||
logs: []string{"a", "b"},
|
||||
expected: "a\nb",
|
||||
},
|
||||
{
|
||||
max: 2,
|
||||
logs: []string{"a", "b", "c"},
|
||||
expected: "b\nc",
|
||||
},
|
||||
{
|
||||
max: 2,
|
||||
logs: []string{"a", "b", "c", "d"},
|
||||
expected: "c\nd",
|
||||
},
|
||||
} {
|
||||
b := NewLogBuffer(test.max)
|
||||
for _, log := range test.logs {
|
||||
b.Push(&types.KernelLog{Message: log})
|
||||
}
|
||||
got := b.String()
|
||||
if test.expected != got {
|
||||
t.Errorf("case %d: expected %q, got %q", c+1, test.expected, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatch(t *testing.T) {
|
||||
max := 4
|
||||
for c, test := range []struct {
|
||||
logs []string
|
||||
exprs []string
|
||||
expected [][]string
|
||||
}{
|
||||
{
|
||||
// Buffer not full
|
||||
logs: []string{"a1", "b2"},
|
||||
exprs: []string{
|
||||
"a1", // Not including the last line, should not match
|
||||
"b1", // Not match
|
||||
"b2", // match
|
||||
`\w{2}`, // Regexp should work
|
||||
"a1\nb2", // Including the last line, should match
|
||||
`a1b2`, // No new line, should not match
|
||||
},
|
||||
expected: [][]string{{}, {}, {"b2"}, {"b2"}, {"a1", "b2"}, {}},
|
||||
},
|
||||
{
|
||||
// Buffer full
|
||||
logs: []string{"a1", "b2", "c3", "d4", "e5"},
|
||||
exprs: []string{
|
||||
"(?s)a1.+", // Rotate out, should not match
|
||||
`[a-z]\d\n[a-z]\d`, // New line should work, and only the one contains the last line should match
|
||||
`[a-z]\d`, // Multiple match, only the one contains the last line should match
|
||||
},
|
||||
expected: [][]string{{}, {"d4", "e5"}, {"e5"}},
|
||||
},
|
||||
} {
|
||||
b := NewLogBuffer(max)
|
||||
for _, log := range test.logs {
|
||||
b.Push(&types.KernelLog{Message: log})
|
||||
}
|
||||
for i, expr := range test.exprs {
|
||||
kLogs := b.Match(expr)
|
||||
got := []string{}
|
||||
for _, kLog := range kLogs {
|
||||
got = append(got, kLog.Message)
|
||||
}
|
||||
if !reflect.DeepEqual(test.expected[i], got) {
|
||||
t.Errorf("case %d.%d: expected %v, got %v", c+1, i+1, test.expected[i], got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package translator
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"k8s.io/node-problem-detector/pkg/kernelmonitor/types"
|
||||
)
|
||||
|
||||
// Translator translates a log line into types.KernelLog, so that kernel monitor
|
||||
// could parse it whatever the original format is.
|
||||
type Translator interface {
|
||||
// Translate translates one log line into types.KernelLog.
|
||||
Translate(string) (*types.KernelLog, error)
|
||||
}
|
||||
|
||||
// defaultTranslator works well for ubuntu and debian, but may not work well with
|
||||
// other os distros. However it is easy to add a new translator for new os distro.
|
||||
type defaultTranslator struct{}
|
||||
|
||||
// NewDefaultTranslator creates a default translator.
|
||||
func NewDefaultTranslator() Translator {
|
||||
return &defaultTranslator{}
|
||||
}
|
||||
|
||||
func (t *defaultTranslator) Translate(line string) (*types.KernelLog, error) {
|
||||
timestr, message, err := parseLine(line)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
timestamp, err := parseTimestamp(timestr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &types.KernelLog{
|
||||
Timestamp: timestamp,
|
||||
Message: message,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseLine(line string) (string, string, error) {
|
||||
// Example line: Jan 1 00:00:00 hostname kernel: [0.000000] component: log message
|
||||
timestampPrefix := "kernel: ["
|
||||
timestampSuffix := "]"
|
||||
idx := strings.Index(line, timestampPrefix)
|
||||
if idx == -1 {
|
||||
return "", "", fmt.Errorf("can't find timestamp prefix %q in line %q", timestampPrefix, line)
|
||||
}
|
||||
line = line[idx+len(timestampPrefix):]
|
||||
|
||||
idx = strings.Index(line, timestampSuffix)
|
||||
if idx == -1 {
|
||||
return "", "", fmt.Errorf("can't find timestamp suffix %q in line %q", timestampSuffix, line)
|
||||
}
|
||||
|
||||
timestamp := strings.Trim(line[:idx], " ")
|
||||
message := strings.Trim(line[idx+1:], " ")
|
||||
|
||||
return timestamp, message, nil
|
||||
}
|
||||
|
||||
func parseTimestamp(timestamp string) (int64, error) {
|
||||
f, err := strconv.ParseFloat(timestamp, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// seconds to microseconds
|
||||
return int64(f * 1000000), nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package translator
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDefaultTranslator(t *testing.T) {
|
||||
tr := NewDefaultTranslator()
|
||||
|
||||
testCases := []struct {
|
||||
input string
|
||||
err bool
|
||||
timestamp int64
|
||||
message string
|
||||
}{
|
||||
{
|
||||
input: "Jan 1 00:00:00 hostname kernel: [9.999999] component: log message",
|
||||
timestamp: 9999999,
|
||||
message: "component: log message",
|
||||
},
|
||||
{
|
||||
input: "Jan 1 00:00:00 hostname kernel: [9.999999]",
|
||||
timestamp: 9999999,
|
||||
message: "",
|
||||
},
|
||||
{
|
||||
input: "Jan 1 00:00:00 hostname kernel: [9.999999 component: log message",
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
input: "Jan 1 00:00:00 hostname user: [9.999999] component: log message",
|
||||
err: true,
|
||||
},
|
||||
}
|
||||
|
||||
for c, test := range testCases {
|
||||
log, err := tr.Translate(test.input)
|
||||
if test.err {
|
||||
if err == nil {
|
||||
t.Errorf("case %d: expect error should occur, got %+v, %v", c+1, log, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if test.timestamp != log.Timestamp || test.message != log.Message {
|
||||
t.Errorf("case %d: expect timestamp: %d, message: %q; got %+v", c+1, test.timestamp, test.message, log)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package types
|
||||
|
||||
// KernelLog is the log item returned by translator. It's very easy to extend this
|
||||
// to support other log monitoring, such as docker log monitoring.
|
||||
type KernelLog struct {
|
||||
Timestamp int64 // microseconds since kernel boot
|
||||
Message string
|
||||
}
|
||||
|
||||
// Type is the type of the kernel problem.
|
||||
type Type string
|
||||
|
||||
const (
|
||||
// Temp means the kernel problem is temporary, only need to report an event.
|
||||
Temp Type = "temporary"
|
||||
// Perm means the kernel problem is permanent, need to change the node condition.
|
||||
Perm Type = "permanent"
|
||||
)
|
||||
|
||||
// Rule describes how kernel monitor should analyze the kernel log.
|
||||
type Rule struct {
|
||||
// Type is the type of matched kernel problem.
|
||||
Type Type `json:"type"`
|
||||
// Reason is the short reason of the kernel problem.
|
||||
Reason string `json:"reason"`
|
||||
// Pattern is the regular expression to match the kernel problem in kernel log.
|
||||
// Notice that the pattern must match to the end of the line.
|
||||
Pattern string `json:"pattern"`
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package util
|
||||
|
||||
// Tomb is used to control the lifecycle of a goroutine.
|
||||
type Tomb struct {
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// NewTomb creates a new tomb.
|
||||
func NewTomb() *Tomb {
|
||||
return &Tomb{
|
||||
stop: make(chan struct{}),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Stop is used to stop the goroutine outside.
|
||||
func (t *Tomb) Stop() {
|
||||
close(t.stop)
|
||||
<-t.done
|
||||
}
|
||||
|
||||
// Stopping is used by the goroutine to tell whether it should stop.
|
||||
func (t *Tomb) Stopping() <-chan struct{} {
|
||||
return t.stop
|
||||
}
|
||||
|
||||
// Done is used by the goroutine to inform that it has stopped.
|
||||
func (t *Tomb) Done() {
|
||||
close(t.done)
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTomb(t *testing.T) {
|
||||
tomb := NewTomb()
|
||||
workflow := []string{}
|
||||
expected := []string{"stop", "stopping", "stopped"}
|
||||
go func() {
|
||||
defer tomb.Done()
|
||||
<-tomb.Stopping()
|
||||
workflow = append(workflow, "stopping")
|
||||
}()
|
||||
workflow = append(workflow, "stop")
|
||||
tomb.Stop()
|
||||
workflow = append(workflow, "stopped")
|
||||
if !reflect.DeepEqual(workflow, expected) {
|
||||
t.Errorf("expected workflow %v, got %v", expected, workflow)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user