From ecaa61e7d3c58a62f560e3d08200f237283982a7 Mon Sep 17 00:00:00 2001
From: Zhen Wang <zhenw@google.com>
Date: Mon, 19 Nov 2018 16:07:57 -0800
Subject: [PATCH] Detect readonly filesystem

---
 config/kernel-monitor.json                     | 18 +++++++++++++-----
 deployment/node-problem-detector-config.yaml   | 18 +++++++++++++-----
 .../problems/readonly_filesystem               |  1 +
 3 files changed, 27 insertions(+), 10 deletions(-)
 create mode 100644 test/kernel_log_generator/problems/readonly_filesystem

diff --git a/config/kernel-monitor.json b/config/kernel-monitor.json
index 9796a839..aa2dd4a5 100644
--- a/config/kernel-monitor.json
+++ b/config/kernel-monitor.json
@@ -1,9 +1,6 @@
 {
-	"plugin": "journald",
-	"pluginConfig": {
-		"source": "kernel"
-	},
-	"logPath": "/var/log/journal",
+	"plugin": "kmsg",
+	"logPath": "/dev/kmsg",
 	"lookback": "5m",
 	"bufferSize": 10,
 	"source": "kernel-monitor",
@@ -12,6 +9,11 @@
 			"type": "KernelDeadlock",
 			"reason": "KernelHasNoDeadlock",
 			"message": "kernel has no deadlock"
+		},
+		{
+			"type": "ReadonlyFilesystem",
+			"reason": "FilesystemIsNotReadOnly",
+			"message": "Filesystem is not read-only"
 		}
 	],
 	"rules": [
@@ -51,6 +53,12 @@
 			"condition": "KernelDeadlock",
 			"reason": "DockerHung",
 			"pattern": "task docker:\\w+ blocked for more than \\w+ seconds\\."
+		},
+		{
+			"type": "permanent",
+			"condition": "ReadonlyFilesystem",
+			"reason": "FilesystemIsReadOnly",
+			"pattern": "Remounting filesystem read-only"
 		}
 	]
 }
diff --git a/deployment/node-problem-detector-config.yaml b/deployment/node-problem-detector-config.yaml
index 84ac4e98..69cdd277 100644
--- a/deployment/node-problem-detector-config.yaml
+++ b/deployment/node-problem-detector-config.yaml
@@ -2,11 +2,8 @@ apiVersion: v1
 data:
   kernel-monitor.json: |
     {
-        "plugin": "journald",
-        "pluginConfig": {
-            "source": "kernel"
-        },
-        "logPath": "/var/log/journal",
+        "plugin": "kmsg",
+        "logPath": "/dev/kmsg",
         "lookback": "5m",
         "bufferSize": 10,
         "source": "kernel-monitor",
@@ -15,6 +12,11 @@ data:
                 "type": "KernelDeadlock",
                 "reason": "KernelHasNoDeadlock",
                 "message": "kernel has no deadlock"
+            },
+            {
+                "type": "ReadonlyFilesystem",
+                "reason": "FilesystemIsReadOnly",
+                "message": "Filesystem is read-only"
             }
         ],
         "rules": [
@@ -54,6 +56,12 @@ data:
                 "condition": "KernelDeadlock",
                 "reason": "DockerHung",
                 "pattern": "task docker:\\w+ blocked for more than \\w+ seconds\\."
+            },
+            {
+                "type": "permanent",
+                "condition": "ReadonlyFilesystem",
+                "reason": "FilesystemIsReadOnly",
+                "pattern": "Remounting filesystem read-only"
             }
         ]
     }
diff --git a/test/kernel_log_generator/problems/readonly_filesystem b/test/kernel_log_generator/problems/readonly_filesystem
new file mode 100644
index 00000000..1bcc1c08
--- /dev/null
+++ b/test/kernel_log_generator/problems/readonly_filesystem
@@ -0,0 +1 @@
+EXT4-fs (sda1): Remounting filesystem read-only