This PR:

1) Add lookback support in kernel monitor. After started, Kernel monitor
will check some old logs to detect problems which happened before last
node reboot.
2) Add `lookback` and `startPattern` in kernel monitor configuration.
  * `lookback` specifies how long time kernel monitor should look back.
  * `startPattern` specifies which log indicates the node is started.
  kernel monitor will clear all current node conditions once it finds
  a node start log. This makes sure that old problems won't change the
  node condition.
3) Add support for kernel panic monitoring, the null pointer and divide
0 kernel panic will be surfaced as event. Usually kernel monitor will
report these events during looking back phase.

pkg/kernelmonitor/translator/translator.go

@@ -18,8 +18,8 @@ package translator
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
+	"time"
 
 	"k8s.io/node-problem-detector/pkg/kernelmonitor/types"
 )
@@ -41,11 +41,7 @@ func NewDefaultTranslator() Translator {
 }
 
 func (t *defaultTranslator) Translate(line string) (*types.KernelLog, error) {
-	timestr, message, err := parseLine(line)
-	if err != nil {
-		return nil, err
-	}
-	timestamp, err := parseTimestamp(timestr)
+	timestamp, message, err := t.parseLine(line)
 	if err != nil {
 		return nil, err
 	}
@@ -55,32 +51,35 @@ func (t *defaultTranslator) Translate(line string) (*types.KernelLog, error) {
 	}, nil
 }
 
-func parseLine(line string) (string, string, error) {
+var (
+	timestampLen  = 15
+	messagePrefix = "]"
+)
+
+func (t *defaultTranslator) parseLine(line string) (time.Time, string, error) {
+	// Trim the spaces to make sure timestamp could be found
+	line = strings.TrimSpace(line)
+	if len(line) < timestampLen {
+		return time.Time{}, "", fmt.Errorf("the line is too short: %q", line)
+	}
 	// Example line: Jan  1 00:00:00 hostname kernel: [0.000000] component: log message
-	timestampPrefix := "kernel: ["
-	timestampSuffix := "]"
-	idx := strings.Index(line, timestampPrefix)
-	if idx == -1 {
-		return "", "", fmt.Errorf("can't find timestamp prefix %q in line %q", timestampPrefix, line)
+	now := time.Now()
+	// There is no time zone information in kernel log timestamp, apply the current time
+	// zone.
+	timestamp, err := time.ParseInLocation(time.Stamp, line[:timestampLen], time.Local)
+	if err != nil {
+		return time.Time{}, "", fmt.Errorf("error parsing timestamp in line %q: %v", line, err)
 	}
-	line = line[idx+len(timestampPrefix):]
+	// There is no year information in kernel log timestamp, apply the current year.
+	// This could go wrong during looking back phase after kernel monitor is started,
+	// and the old logs are generated in old year.
+	timestamp = timestamp.AddDate(now.Year(), 0, 0)
 
-	idx = strings.Index(line, timestampSuffix)
-	if idx == -1 {
-		return "", "", fmt.Errorf("can't find timestamp suffix %q in line %q", timestampSuffix, line)
+	loc := strings.Index(line, messagePrefix)
+	if loc == -1 {
+		return timestamp, "", fmt.Errorf("can't find message prefix %q in line %q", messagePrefix, line)
 	}
-
-	timestamp := strings.Trim(line[:idx], " ")
-	message := strings.Trim(line[idx+1:], " ")
+	message := strings.Trim(line[loc+1:], " ")
 
 	return timestamp, message, nil
 }
-
-func parseTimestamp(timestamp string) (int64, error) {
-	f, err := strconv.ParseFloat(timestamp, 64)
-	if err != nil {
-		return 0, err
-	}
-	// seconds to microseconds
-	return int64(f * 1000000), nil
-}

pkg/kernelmonitor/translator/translator_test.go

@@ -18,33 +18,32 @@ package translator
 
 import (
 	"testing"
+	"time"
 )
 
 func TestDefaultTranslator(t *testing.T) {
 	tr := NewDefaultTranslator()
-
+	year := time.Now().Year()
 	testCases := []struct {
 		input     string
 		err       bool
-		timestamp int64
+		timestamp time.Time
 		message   string
 	}{
 		{
-			input:     "Jan  1 00:00:00 hostname kernel: [9.999999] component: log message",
-			timestamp: 9999999,
+			input:     "May  1 12:23:45 hostname kernel: [0.000000] component: log message",
+			timestamp: time.Date(year, time.May, 1, 12, 23, 45, 0, time.Local),
 			message:   "component: log message",
 		},
 		{
-			input:     "Jan  1 00:00:00 hostname kernel: [9.999999]",
-			timestamp: 9999999,
+			// no log message
+			input:     "May 21 12:23:45 hostname kernel: [9.999999]",
+			timestamp: time.Date(year, time.May, 21, 12, 23, 45, 0, time.Local),
 			message:   "",
 		},
 		{
-			input: "Jan  1 00:00:00 hostname kernel: [9.999999 component: log message",
-			err:   true,
-		},
-		{
-			input: "Jan  1 00:00:00 hostname user: [9.999999] component: log message",
+			// the right square bracket is missing
+			input: "May 21 12:23:45 hostname kernel: [9.999999 component: log message",
 			err:   true,
 		},
 	}
@@ -58,7 +57,7 @@ func TestDefaultTranslator(t *testing.T) {
 			continue
 		}
 		if test.timestamp != log.Timestamp || test.message != log.Message {
-			t.Errorf("case %d: expect timestamp: %d, message: %q; got %+v", c+1, test.timestamp, test.message, log)
+			t.Errorf("case %d: expect %v, %q; got %v, %q", c+1, test.timestamp, test.message, log.Timestamp, log.Message)
 		}
 	}
 }