mirror of
https://github.com/kubernetes/node-problem-detector.git
synced 2026-08-19 04:06:24 +00:00
fix(logwatchers/kmsg): don't close the old parser twice when Stop() interrupts a restart
The restart path closes the failed parser before retrying. If stopping is signaled during the retry wait, watchLoop's deferred cleanup closed the same parser again, logging a spurious 'file already closed' error at shutdown. Clear the reference after closing and nil-check the defer.
This commit is contained in:
@@ -94,8 +94,11 @@ func (k *kernelLogWatcher) Stop() {
|
||||
func (k *kernelLogWatcher) watchLoop() {
|
||||
kmsgs := k.kmsgParser.Parse()
|
||||
defer func() {
|
||||
if err := k.kmsgParser.Close(); err != nil {
|
||||
klog.Errorf("Failed to close kmsg parser: %v", err)
|
||||
// kmsgParser is nil when stopping interrupted a restart.
|
||||
if k.kmsgParser != nil {
|
||||
if err := k.kmsgParser.Close(); err != nil {
|
||||
klog.Errorf("Failed to close kmsg parser: %v", err)
|
||||
}
|
||||
}
|
||||
close(k.logCh)
|
||||
k.tomb.Done()
|
||||
@@ -110,10 +113,12 @@ func (k *kernelLogWatcher) watchLoop() {
|
||||
if !ok {
|
||||
klog.Error("Kmsg channel closed, attempting to restart kmsg parser")
|
||||
|
||||
// Close the old parser
|
||||
// Close the old parser and clear the reference so the
|
||||
// deferred cleanup doesn't close it a second time.
|
||||
if err := k.kmsgParser.Close(); err != nil {
|
||||
klog.Errorf("Failed to close kmsg parser: %v", err)
|
||||
}
|
||||
k.kmsgParser = nil
|
||||
|
||||
// Try to restart. retryCreateParser() waits between attempts.
|
||||
var restarted bool
|
||||
|
||||
@@ -468,6 +468,59 @@ func TestWatcherRateLimitsRestarts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopDuringRestartClosesOldParserOnce verifies that when Stop() arrives
|
||||
// while the watcher is in the restart path, the already-closed old parser is
|
||||
// not closed a second time by watchLoop's deferred cleanup.
|
||||
func TestStopDuringRestartClosesOldParserOnce(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
// Closing the channel after sending drives watchLoop into the restart path.
|
||||
mock := &mockKmsgParser{
|
||||
kmsgs: []kmsgparser.Message{{Message: "msg", Timestamp: now}},
|
||||
closeAfterSend: true,
|
||||
}
|
||||
|
||||
factoryCalled := make(chan struct{}, 1)
|
||||
w := &kernelLogWatcher{
|
||||
cfg: types.WatcherConfig{},
|
||||
startTime: now.Add(-time.Minute),
|
||||
tomb: tomb.NewTomb(),
|
||||
logCh: make(chan *logtypes.Log, 100),
|
||||
kmsgParser: mock,
|
||||
newParser: func() (kmsgparser.Parser, error) {
|
||||
select {
|
||||
case factoryCalled <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
// Keep the watcher in the retry loop until Stop() is called.
|
||||
return nil, fmt.Errorf("kmsg unavailable")
|
||||
},
|
||||
}
|
||||
|
||||
logCh, err := w.Watch()
|
||||
assert.NoError(t, err)
|
||||
<-logCh
|
||||
|
||||
// Wait until watchLoop has entered the restart path.
|
||||
select {
|
||||
case <-factoryCalled:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timeout waiting for restart attempt")
|
||||
}
|
||||
|
||||
w.Stop()
|
||||
|
||||
select {
|
||||
case _, ok := <-logCh:
|
||||
assert.False(t, ok, "log channel should be closed after Stop()")
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timeout waiting for log channel to close after Stop()")
|
||||
}
|
||||
|
||||
assert.Equal(t, 1, mock.CloseCallCount(),
|
||||
"old parser must be closed exactly once, not again by watchLoop's defer")
|
||||
}
|
||||
|
||||
// TestStopDoesNotDeadlockWhenLogChannelFull verifies that Stop() returns even
|
||||
// when logCh is full and nobody is draining it.
|
||||
func TestStopDoesNotDeadlockWhenLogChannelFull(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user