From 6106adc2b2c2d196a3203b5234fef8cfcd42d3de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Socho=C5=84?= <kaszpir@gmail.com>
Date: Fri, 18 Feb 2022 20:29:37 +0100
Subject: [PATCH] Update runbooks/general

---
 content/runbooks/general/InfoInhibitor.md     | 27 ++++++++++++++++++
 .../general/NodeNetworkInterfaceFlapping.md   | 24 ++++++++++++++++
 content/runbooks/general/Watchdog.md          | 28 +++++++++++++++++++
 3 files changed, 79 insertions(+)
 create mode 100644 content/runbooks/general/InfoInhibitor.md
 create mode 100644 content/runbooks/general/NodeNetworkInterfaceFlapping.md
 create mode 100644 content/runbooks/general/Watchdog.md

diff --git a/content/runbooks/general/InfoInhibitor.md b/content/runbooks/general/InfoInhibitor.md
new file mode 100644
index 0000000..87fa6ac
--- /dev/null
+++ b/content/runbooks/general/InfoInhibitor.md
@@ -0,0 +1,27 @@
+---
+title: Info Inhibitor
+weight: 20
+---
+
+# InfoInhibitor
+
+## Meaning
+
+This is an alert that is used to inhibit info alerts.
+
+By themselves, the info-level alerts are sometimes very noisy,
+but they are relevant when combined with other alerts.
+          
+## Impact
+
+## Diagnosis
+
+This alert fires whenever there's a `severity="info"` alert,
+and stops firing when another alert with a severity of `warning` or
+`critical` starts firing on the same namespace.
+
+
+## Mitigation
+
+This alert should be routed to a null receiver and configured to inhibit
+alerts with `severity="info"`.
diff --git a/content/runbooks/general/NodeNetworkInterfaceFlapping.md b/content/runbooks/general/NodeNetworkInterfaceFlapping.md
new file mode 100644
index 0000000..7673719
--- /dev/null
+++ b/content/runbooks/general/NodeNetworkInterfaceFlapping.md
@@ -0,0 +1,24 @@
+---
+title: Node Network Interface Flapping
+weight: 20
+---
+
+# NodeNetworkInterfaceFlapping
+
+## Meaning
+
+Network interface is often changing its status
+
+## Impact
+
+Applications on the node may no longer be able to operate with other services.
+Network attached storage performance issues or even data loss.
+
+## Diagnosis
+
+Investigate networkng issues on the node and to connected hardware.
+Check physical cables, check networking firewall rules and so on.
+
+## Mitigation
+
+Cordon and drain node to migrate apps from it.
diff --git a/content/runbooks/general/Watchdog.md b/content/runbooks/general/Watchdog.md
new file mode 100644
index 0000000..4f377db
--- /dev/null
+++ b/content/runbooks/general/Watchdog.md
@@ -0,0 +1,28 @@
+---
+title: Watchdog
+weight: 20
+---
+
+# Watchdog
+
+## Meaning
+
+This is an alert meant to ensure that the entire alerting pipeline is functional.
+This alert is always firing, therefore it should always be firing in Alertmanager
+and always fire against a receiver.
+
+## Impact
+
+If not firing then it should alert external systems that this alerting system
+is no longer working.
+
+## Diagnosis
+
+Misconfigured alertmanager, bad credentials, bad endpoint, firewalls..
+Check alertmanager logs.
+
+## Mitigation
+
+There are integrations with various notification
+mechanisms that send a notification when this alert is not firing.
+For example the `DeadMansSnitch` integration in PagerDuty.