Merge pull request #1323 from hakman/precompile-log-buffer-regexps

Precompile log buffer regular expressions

pkg/systemlogmonitor/config.go

@@ -59,13 +59,14 @@ func (mc *MonitorConfig) ApplyDefaultConfiguration() {
 	}
 }
 
-// ValidateRules verifies whether the regular expressions in the rules are valid.
-func (mc MonitorConfig) ValidateRules() error {
-	for _, rule := range mc.Rules {
-		_, err := regexp.Compile(rule.Pattern)
+func (mc MonitorConfig) compileRules() ([]*regexp.Regexp, error) {
+	patterns := make([]*regexp.Regexp, len(mc.Rules))
+	for i, rule := range mc.Rules {
+		pattern, err := CompilePattern(rule.Pattern)
 		if err != nil {
-			return err
+			return nil, err
 		}
+		patterns[i] = pattern
 	}
-	return nil
+	return patterns, nil
 }

pkg/systemlogmonitor/log_buffer.go

@@ -28,7 +28,7 @@ type LogBuffer interface {
 	// Push pushes log into the log buffer.
 	Push(*types.Log)
 	// Match with regular expression in the log buffer.
-	Match(string) []*types.Log
+	Match(*regexp.Regexp) []*types.Log
 	// String returns a concatenated string of the buffered logs.
 	String() string
 }
@@ -39,8 +39,6 @@ type logBuffer struct {
 	msg     []string
 	max     int
 	current int
-	// regexps caches compiled regular expressions.
-	regexps map[string]*regexp.Regexp
 }
 
 // NewLogBuffer creates log buffer with max line number limit. Because we only match logs
@@ -49,26 +47,28 @@ type logBuffer struct {
 // lines of patterns we support.
 func NewLogBuffer(maxLines int) *logBuffer {
 	return &logBuffer{
-		buffer:  make([]*types.Log, maxLines),
-		msg:     make([]string, maxLines),
-		max:     maxLines,
-		regexps: make(map[string]*regexp.Regexp),
+		buffer: make([]*types.Log, maxLines),
+		msg:    make([]string, maxLines),
+		max:    maxLines,
 	}
 }
 
+// CompilePattern compiles a log buffer pattern that must match to the end of
+// the buffered logs.
+func CompilePattern(expr string) (*regexp.Regexp, error) {
+	if _, err := regexp.Compile(expr); err != nil {
+		return nil, err
+	}
+	return regexp.Compile(expr + `\z`)
+}
+
 func (b *logBuffer) Push(log *types.Log) {
 	b.buffer[b.current%b.max] = log
 	b.msg[b.current%b.max] = log.Message
 	b.current++
 }
 
-func (b *logBuffer) Match(expr string) []*types.Log {
-	// The expression should be checked outside, and it must match to the end.
-	reg, ok := b.regexps[expr]
-	if !ok {
-		reg = regexp.MustCompile(expr + `\z`)
-		b.regexps[expr] = reg
-	}
+func (b *logBuffer) Match(reg *regexp.Regexp) []*types.Log {
 	log := b.String()
 	loc := reg.FindStringIndex(log)
 	if loc == nil {

pkg/systemlogmonitor/log_buffer_test.go

@@ -61,6 +61,12 @@ func TestPush(t *testing.T) {
 	}
 }
 
+func TestCompilePatternRejectsInvalidExpression(t *testing.T) {
+	if _, err := CompilePattern(`foo\`); err == nil {
+		t.Fatal("expected invalid expression error")
+	}
+}
+
 func TestMatch(t *testing.T) {
 	max := 4
 	for c, test := range []struct {
@@ -97,7 +103,11 @@ func TestMatch(t *testing.T) {
 			b.Push(&types.Log{Message: log})
 		}
 		for i, expr := range test.exprs {
-			logs := b.Match(expr)
+			pattern, err := CompilePattern(expr)
+			if err != nil {
+				t.Fatalf("case %d.%d: failed to compile pattern %q: %v", c+1, i+1, expr, err)
+			}
+			logs := b.Match(pattern)
 			got := []string{}
 			for _, log := range logs {
 				got = append(got, log.Message)
@@ -117,8 +127,12 @@ func BenchmarkMatch(b *testing.B) {
 	// A pattern from the default kernel monitor configuration which does not
 	// match the buffered logs.
 	expr := `task [\S ]+:\w+ blocked for more than \w+ seconds\.`
+	pattern, err := CompilePattern(expr)
+	if err != nil {
+		b.Fatalf("failed to compile pattern %q: %v", expr, err)
+	}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		buf.Match(expr)
+		buf.Match(pattern)
 	}
 }

pkg/systemlogmonitor/log_monitor.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"regexp"
 	"time"
 
 	"k8s.io/klog/v2"
@@ -50,6 +51,7 @@ type logMonitor struct {
 	watcher    watchertypes.LogWatcher
 	buffer     LogBuffer
 	config     MonitorConfig
+	patterns   []*regexp.Regexp
 	conditions []types.Condition
 	logCh      <-chan *systemlogtypes.Log
 	output     chan *types.Status
@@ -73,7 +75,7 @@ func NewLogMonitorOrDie(configPath string) types.Monitor {
 	}
 	// Apply default configurations
 	(&l.config).ApplyDefaultConfiguration()
-	err = l.config.ValidateRules()
+	l.patterns, err = l.config.compileRules()
 	if err != nil {
 		klog.Fatalf("Failed to validate %s matching rules %+v: %v", l.configPath, l.config.Rules, err)
 	}
@@ -152,8 +154,8 @@ func (l *logMonitor) parseLog(log *systemlogtypes.Log) {
 	// Once there is new log, log monitor will push it into the log buffer and try
 	// to match each rule. If any rule is matched, log monitor will report a status.
 	l.buffer.Push(log)
-	for _, rule := range l.config.Rules {
-		matched := l.buffer.Match(rule.Pattern)
+	for i, rule := range l.config.Rules {
+		matched := l.buffer.Match(l.patterns[i])
 		if len(matched) == 0 {
 			continue
 		}