diff --git a/Dockerfile b/Dockerfile index 6d1de439..1b018d20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,4 +50,4 @@ ARG LOGCOUNTER COPY --from=builder /gopath/src/k8s.io/node-problem-detector/bin/health-checker /gopath/src/k8s.io/node-problem-detector/${LOGCOUNTER} /home/kubernetes/bin/ COPY --from=builder /gopath/src/k8s.io/node-problem-detector/config/ /config -ENTRYPOINT ["/node-problem-detector", "--config.system-log-monitor=/config/kernel-monitor.json"] +ENTRYPOINT ["/node-problem-detector", "--config.system-log-monitor=/config/kernel-monitor.json,/config/readonly-monitor.json"] diff --git a/config/kernel-monitor.json b/config/kernel-monitor.json index 832a063b..9ba249c0 100644 --- a/config/kernel-monitor.json +++ b/config/kernel-monitor.json @@ -10,11 +10,6 @@ "type": "KernelDeadlock", "reason": "KernelHasNoDeadlock", "message": "kernel has no deadlock" - }, - { - "type": "ReadonlyFilesystem", - "reason": "FilesystemIsNotReadOnly", - "message": "Filesystem is not read-only" } ], "rules": [ @@ -68,12 +63,6 @@ "condition": "KernelDeadlock", "reason": "DockerHung", "pattern": "task docker:\\w+ blocked for more than \\w+ seconds\\." - }, - { - "type": "permanent", - "condition": "ReadonlyFilesystem", - "reason": "FilesystemIsReadOnly", - "pattern": "Remounting filesystem read-only" } ] } diff --git a/config/readonly-monitor.json b/config/readonly-monitor.json new file mode 100644 index 00000000..256e431e --- /dev/null +++ b/config/readonly-monitor.json @@ -0,0 +1,23 @@ +{ + "plugin": "kmsg", + "logPath": "/dev/kmsg", + "lookback": "5m", + "bufferSize": 10, + "source": "readonly-monitor", + "metricsReporting": true, + "conditions": [ + { + "type": "ReadonlyFilesystem", + "reason": "FilesystemIsNotReadOnly", + "message": "Filesystem is not read-only" + } + ], + "rules": [ + { + "type": "permanent", + "condition": "ReadonlyFilesystem", + "reason": "FilesystemIsReadOnly", + "pattern": "Remounting filesystem read-only" + } + ] +} diff --git a/config/systemd/node-problem-detector-metric-only.service b/config/systemd/node-problem-detector-metric-only.service index b84206b6..76cd5ad8 100644 --- a/config/systemd/node-problem-detector-metric-only.service +++ b/config/systemd/node-problem-detector-metric-only.service @@ -8,7 +8,7 @@ Restart=always RestartSec=10 ExecStart=/home/kubernetes/bin/node-problem-detector --v=2 --logtostderr --enable-k8s-exporter=false \ --exporter.stackdriver=/home/kubernetes/node-problem-detector/config/exporter/stackdriver-exporter.json \ - --config.system-log-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor.json,/home/kubernetes/node-problem-detector/config/docker-monitor.json,/home/kubernetes/node-problem-detector/config/systemd-monitor.json \ + --config.system-log-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor.json,/home/kubernetes/node-problem-detector/config/readonly-monitor.json,/home/kubernetes/node-problem-detector/config/docker-monitor.json,/home/kubernetes/node-problem-detector/config/systemd-monitor.json \ --config.custom-plugin-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor-counter.json,/home/kubernetes/node-problem-detector/config/systemd-monitor-counter.json \ --config.system-stats-monitor=/home/kubernetes/node-problem-detector/config/system-stats-monitor.json,/home/kubernetes/node-problem-detector/config/net-cgroup-system-stats-monitor.json diff --git a/deployment/node-problem-detector-config.yaml b/deployment/node-problem-detector-config.yaml index 6e9e2486..e80aac8b 100644 --- a/deployment/node-problem-detector-config.yaml +++ b/deployment/node-problem-detector-config.yaml @@ -64,6 +64,30 @@ data: } ] } + readonly-monitor.json: | + { + "plugin": "kmsg", + "logPath": "/dev/kmsg", + "lookback": "5m", + "bufferSize": 10, + "source": "readonly-monitor", + "metricsReporting": true, + "conditions": [ + { + "type": "ReadonlyFilesystem", + "reason": "FilesystemIsNotReadOnly", + "message": "Filesystem is not read-only" + } + ], + "rules": [ + { + "type": "permanent", + "condition": "ReadonlyFilesystem", + "reason": "FilesystemIsReadOnly", + "pattern": "Remounting filesystem read-only" + } + ] + } docker-monitor.json: | { "plugin": "journald", diff --git a/deployment/node-problem-detector-healthchecker.yaml b/deployment/node-problem-detector-healthchecker.yaml index 1b452cad..f7f4fc50 100644 --- a/deployment/node-problem-detector-healthchecker.yaml +++ b/deployment/node-problem-detector-healthchecker.yaml @@ -28,7 +28,7 @@ spec: command: - /node-problem-detector - --logtostderr - - --config.system-log-monitor=/config/kernel-monitor.json,/config/docker-monitor.json + - --config.system-log-monitor=/config/kernel-monitor.json,/config/readonly-monitor.json,/config/docker-monitor.json - --config.custom-plugin-monitor=/config/health-checker-kubelet.json image: registry.k8s.io/node-problem-detector/node-problem-detector:v0.8.19 resources: @@ -86,6 +86,8 @@ spec: items: - key: kernel-monitor.json path: kernel-monitor.json + - key: readonly-monitor.json + path: readonly-monitor.json - key: docker-monitor.json path: docker-monitor.json - name: machine-id diff --git a/deployment/node-problem-detector.yaml b/deployment/node-problem-detector.yaml index bcd0e8cc..a3614e96 100644 --- a/deployment/node-problem-detector.yaml +++ b/deployment/node-problem-detector.yaml @@ -28,7 +28,7 @@ spec: command: - /node-problem-detector - --logtostderr - - --config.system-log-monitor=/config/kernel-monitor.json,/config/docker-monitor.json + - --config.system-log-monitor=/config/kernel-monitor.json,/config/readonly-monitor.json,/config/docker-monitor.json image: registry.k8s.io/node-problem-detector/node-problem-detector:v0.8.19 resources: limits: @@ -78,6 +78,8 @@ spec: items: - key: kernel-monitor.json path: kernel-monitor.json + - key: readonly-monitor.json + path: readonly-monitor.json - key: docker-monitor.json path: docker-monitor.json tolerations: diff --git a/test/build.sh b/test/build.sh index 6ffd12f0..6a351cf5 100755 --- a/test/build.sh +++ b/test/build.sh @@ -97,6 +97,7 @@ function build-npd-custom-flags() { local -r kube_home="/home/kubernetes" local -r km_config="${kube_home}/node-problem-detector/config/kernel-monitor.json" + local -r rm_config="${kube_home}/node-problem-detector/config/readonly-monitor.json" local -r dm_config="${kube_home}/node-problem-detector/config/docker-monitor.json" local -r sm_config="${kube_home}/node-problem-detector/config/systemd-monitor.json" @@ -105,7 +106,7 @@ function build-npd-custom-flags() { flags="--v=2" flags+=" --logtostderr" - flags+=" --config.system-log-monitor=${km_config},${dm_config},${sm_config}" + flags+=" --config.system-log-monitor=${km_config},${rm_config},${dm_config},${sm_config}" flags+=" --config.custom-plugin-monitor=${custom_km_config},${custom_sm_config}" flags+=" --port=20256"