From 651251b5ff918f0f7ef543c592594a4b2d254bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Socho=C5=84?= Date: Fri, 18 Feb 2022 20:15:26 +0100 Subject: [PATCH 1/9] Update runbooks/node except NodeNetworkInterfaceFlapping because this is from /general/ --- .../node/NodeClockNotSynchronising.md | 22 +++++++++++++ .../runbooks/node/NodeClockSkewDetected.md | 33 +++++++++++++++++++ .../runbooks/node/NodeFileDescriptorLimit.md | 7 +++- .../node/NodeFilesystemAlmostOutOfFiles.md | 11 ++++--- .../node/NodeFilesystemAlmostOutOfSpace.md | 13 ++++---- .../node/NodeFilesystemFilesFillingUp.md | 13 +++++--- .../node/NodeFilesystemSpaceFillingUp.md | 29 ++++++++++------ .../NodeHighNumberConntrackEntriesUsed.md | 25 ++++++++++++++ .../runbooks/node/NodeNetworkReceiveErrs.md | 24 ++++++++++++++ .../runbooks/node/NodeNetworkTransmitErrs.md | 24 ++++++++++++++ content/runbooks/node/NodeRAIDDegraded.md | 13 ++++++-- content/runbooks/node/NodeRAIDDiskFailure.md | 8 +++++ .../node/NodeTextFileCollectorScrapeError.md | 22 +++++++++++++ 13 files changed, 217 insertions(+), 27 deletions(-) create mode 100644 content/runbooks/node/NodeClockNotSynchronising.md create mode 100644 content/runbooks/node/NodeClockSkewDetected.md create mode 100644 content/runbooks/node/NodeHighNumberConntrackEntriesUsed.md create mode 100644 content/runbooks/node/NodeNetworkReceiveErrs.md create mode 100644 content/runbooks/node/NodeNetworkTransmitErrs.md create mode 100644 content/runbooks/node/NodeRAIDDiskFailure.md create mode 100644 content/runbooks/node/NodeTextFileCollectorScrapeError.md diff --git a/content/runbooks/node/NodeClockNotSynchronising.md b/content/runbooks/node/NodeClockNotSynchronising.md new file mode 100644 index 0000000..a345b66 --- /dev/null +++ b/content/runbooks/node/NodeClockNotSynchronising.md @@ -0,0 +1,22 @@ +--- +title: Node Clock Not Synchronising +weight: 20 +--- + +# NodeClockNotSynchronising + +## Meaning + +Clock not synchronising. + +## Impact + +Node instability, apps disconencting. + +## Diagnosis + +TODO + +## Mitigation + +See [Node Clok Skew Detected]({{< ref "./NodeClockSkewDetected.md" >}}) diff --git a/content/runbooks/node/NodeClockSkewDetected.md b/content/runbooks/node/NodeClockSkewDetected.md new file mode 100644 index 0000000..d9870fa --- /dev/null +++ b/content/runbooks/node/NodeClockSkewDetected.md @@ -0,0 +1,33 @@ +--- +title: Node Clock Skew Detected +weight: 20 +--- + +# NodeClockSkewDetected + +## Meaning + +Clock skew detected. + +## Impact + +Node instability, apps disconencting. + +## Diagnosis + +TODO + +## Mitigation + +Ensure time synchronization service is running. +Set proper time servers. +Esure to sync time on server start, especially when using +low power mode or hibernation. + +Some resource consuming process can cause issues on given hardware, +so move it to different servers. + +On physical servers check if on-board battery requires replacement. +Check for hardware errors. +Check for firmware updates. +Ensure to use newer hardware (like server mainboard and so on). diff --git a/content/runbooks/node/NodeFileDescriptorLimit.md b/content/runbooks/node/NodeFileDescriptorLimit.md index 9a1d0b5..97ae9c3 100644 --- a/content/runbooks/node/NodeFileDescriptorLimit.md +++ b/content/runbooks/node/NodeFileDescriptorLimit.md @@ -1,3 +1,8 @@ +--- +title: Node File Descriptor Limit +weight: 20 +--- + # NodeFileDescriptorLimit ## Meaning @@ -17,7 +22,7 @@ node. You can open a shell on the node and use the standard Linux utilities to diagnose the issue: -```console +```shell $ NODE_NAME='' $ oc debug "node/$NODE_NAME" diff --git a/content/runbooks/node/NodeFilesystemAlmostOutOfFiles.md b/content/runbooks/node/NodeFilesystemAlmostOutOfFiles.md index ff564da..7f03582 100644 --- a/content/runbooks/node/NodeFilesystemAlmostOutOfFiles.md +++ b/content/runbooks/node/NodeFilesystemAlmostOutOfFiles.md @@ -1,8 +1,13 @@ +--- +title: Node Filesystem Almost Out Of Files +weight: 20 +--- + # NodeFilesystemAlmostOutOfFiles ## Meaning -This alert is similar to the [NodeFilesystemSpaceFillingUp][1] alert, but rather +This alert is similar to the NodeFilesystemSpaceFillingUp alert, but rather than being based on a prediction that a filesystem will run out of inodes in a certain amount of time, it uses simple static thresholds. The alert will fire as at a `warning` level at 5% of available inodes left, and at a `critical` level @@ -18,10 +23,8 @@ of the cluster. ## Diagnosis -Refer to the [NodeFilesystemFilesFillingUp][1] runbook. ## Mitigation -Refer to the [NodeFilesystemFilesFillingUp][1] runbook. +See [Node Filesystem FilesFilling Up]({{< ref "./NodeFilesystemFilesFillingUp.md" >}}) -[1]: ./NodeFilesystemFilesFillingUp.md diff --git a/content/runbooks/node/NodeFilesystemAlmostOutOfSpace.md b/content/runbooks/node/NodeFilesystemAlmostOutOfSpace.md index 5f241cf..3264a0d 100644 --- a/content/runbooks/node/NodeFilesystemAlmostOutOfSpace.md +++ b/content/runbooks/node/NodeFilesystemAlmostOutOfSpace.md @@ -1,8 +1,13 @@ +--- +title: Node Filesystem Almost Out Of Space +weight: 20 +--- + # NodeFilesystemAlmostOutOfSpace ## Meaning -This alert is similar to the [NodeFilesystemSpaceFillingUp][1] alert, but rather +This alert is similar to the NodeFilesystemSpaceFillingUp alert, but rather than being based on a prediction that a filesystem will become full in a certain amount of time, it uses simple static thresholds. The alert will fire as at a `warning` level at 5% space left, and at a `critical` level with 3% space left. @@ -17,10 +22,6 @@ of the cluster. ## Diagnosis -Refer to the [NodeFilesystemSpaceFillingUp][1] runbook. - ## Mitigation -Refer to the [NodeFilesystemSpaceFillingUp][1] runbook. - -[1]: ./NodeFilesystemSpaceFillingUp.md +See [Node Filesystem FilesFilling Up]({{< ref "./NodeFilesystemFilesFillingUp.md" >}}) diff --git a/content/runbooks/node/NodeFilesystemFilesFillingUp.md b/content/runbooks/node/NodeFilesystemFilesFillingUp.md index 62de979..ef09627 100644 --- a/content/runbooks/node/NodeFilesystemFilesFillingUp.md +++ b/content/runbooks/node/NodeFilesystemFilesFillingUp.md @@ -1,8 +1,13 @@ +--- +title: Node Filesystem Files Filling Up +weight: 20 +--- + # NodeFilesystemFilesFillingUp ## Meaning -This alert is similar to the [NodeFilesystemSpaceFillingUp][1] alert, but +This alert is similar to the NodeFilesystemSpaceFillingUp alert, but predicts the filesystem will run out of inodes rather than bytes of storage space. The alert fires at a `critical` level when the filesystem is predicted to run out of available inodes within four hours. @@ -21,7 +26,7 @@ Note the `instance` and `mountpoint` labels from the alert. You can graph the usage history of this filesystem with the following query in the OpenShift web console: -```text +```promql node_filesystem_files_free{ instance="", mountpoint="" @@ -31,7 +36,7 @@ node_filesystem_files_free{ You can also open a debug session on the node and use the standard Linux utilities to locate the source of the usage: -```console +```shell $ MOUNT_POINT='' $ NODE_NAME='' @@ -50,4 +55,4 @@ size. You may be able to solve the problem, or buy time, by increasing size of the storage volume. Otherwise, determine the application that is creating large numbers of files and adjust its configuration or provide it dedicated storage. -[1]: ./NodeFilesystemSpaceFillingUp.md +See [Node Filesystem FilesFilling Up]({{< ref "./NodeFilesystemFilesFillingUp.md" >}}) diff --git a/content/runbooks/node/NodeFilesystemSpaceFillingUp.md b/content/runbooks/node/NodeFilesystemSpaceFillingUp.md index 6b3ec5a..ad2222b 100644 --- a/content/runbooks/node/NodeFilesystemSpaceFillingUp.md +++ b/content/runbooks/node/NodeFilesystemSpaceFillingUp.md @@ -1,3 +1,8 @@ +--- +title: Node Filesystem Space Filling Up +weight: 20 +--- + # NodeFilesystemSpaceFillingUp ## Meaning @@ -11,8 +16,12 @@ time is less than 4h.
Full context -The filesystem on Kubernetes nodes mainly consists of the operating system, [container ephemeral storage][1], container images, and container logs. -Since Kubelet automatically handles [cleaning up old logs][2] and [deleting unused images][3], container ephemeral storage is a common cause of this alert. Although this alert may be triggered before Kubelet's garbage collection kicks in. +The filesystem on Kubernetes nodes mainly consists of the operating system, +[container ephemeral storage][1], container images, and container logs. +Since Kubelet automatically handles [cleaning up old logs][2] and +[deleting unused images][3], container ephemeral storage is a common cause of +this alert. Although this alert may be triggered before Kubelet's garbage +collection kicks in.
@@ -31,7 +40,7 @@ and/or recent offenders. Is this some irregular condition, e.g. a process fails to clean up behind itself or is this organic growth? If monitoring is enabled, the following metric can be watched in PromQL. -```console +```promql node_filesystem_free_bytes ``` @@ -44,7 +53,7 @@ removing unused images solves that issue: Debug the node by accessing the node filesystem: -```console +```shell $ NODE_NAME= $ kubectl -n default debug node/$NODE_NAME $ chroot /host @@ -52,23 +61,23 @@ $ chroot /host Remove dangling images: -```console +```shell # TODO: Command needed ``` Remove unused images: -```console +```shell # TODO: Command needed ``` Exit debug: -```console +```shell $ exit $ exit ``` -[1]: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#local-ephemeral-storage -[2]: https://kubernetes.io/docs/concepts/cluster-administration/logging/ -[3]: https://kubernetes.io/docs/concepts/architecture/garbage-collection/#containers-images +- [1](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#local-ephemeral-storage) +- [2](https://kubernetes.io/docs/concepts/cluster-administration/logging/) +- [3](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#containers-images) diff --git a/content/runbooks/node/NodeHighNumberConntrackEntriesUsed.md b/content/runbooks/node/NodeHighNumberConntrackEntriesUsed.md new file mode 100644 index 0000000..a672a65 --- /dev/null +++ b/content/runbooks/node/NodeHighNumberConntrackEntriesUsed.md @@ -0,0 +1,25 @@ +--- +title: Node High Number Conntrack Entries Used +weight: 20 +--- + +# NodeHighNumberConntrackEntriesUsed + +## Meaning + +Number of conntrack are getting close to the limit. + +## Impact + +When reached the limit then some connections will be dropped, degrading service quality. + +## Diagnosis + +Check current conntrack value on the node. +Check which apps are generating a lot of connections. + +## Mitigation + +Migrate some pods to another nodes. +Bump conntrack limit directly on the node. +Update provisioning scripts to do the same on node start. diff --git a/content/runbooks/node/NodeNetworkReceiveErrs.md b/content/runbooks/node/NodeNetworkReceiveErrs.md new file mode 100644 index 0000000..7c6e543 --- /dev/null +++ b/content/runbooks/node/NodeNetworkReceiveErrs.md @@ -0,0 +1,24 @@ +--- +title: Node Network Receive Errors +weight: 20 +--- + +# NodeNetworkReceiveErrs + +## Meaning + +Network interface is reporting many receive errors. + +## Impact + +Applications on the node may no longer be able to operate with other services. +Network attached storage performance issues or even data loss. + +## Diagnosis + +Investigate networkng issues on the node and to connected hardware. +Check physical cables, check networking firewall rules and so on. + +## Mitigation + +Cordon and drain node to migrate apps from it. diff --git a/content/runbooks/node/NodeNetworkTransmitErrs.md b/content/runbooks/node/NodeNetworkTransmitErrs.md new file mode 100644 index 0000000..d9e182f --- /dev/null +++ b/content/runbooks/node/NodeNetworkTransmitErrs.md @@ -0,0 +1,24 @@ +--- +title: Node Network Transmit Errors +weight: 20 +--- + +# NodeNetworkTransmitErrs + +## Meaning + +Network interface is reporting many transmit errors. + +## Impact + +Applications on the node may no longer be able to operate with other services. +Network attached storage performance issues or even data loss. + +## Diagnosis + +Investigate networkng issues on the node and to connected hardware. +Check physical cables, check networking firewall rules and so on. + +## Mitigation + +Cordon and drain node to migrate apps from it. diff --git a/content/runbooks/node/NodeRAIDDegraded.md b/content/runbooks/node/NodeRAIDDegraded.md index 4adef9c..ced7282 100644 --- a/content/runbooks/node/NodeRAIDDegraded.md +++ b/content/runbooks/node/NodeRAIDDegraded.md @@ -1,7 +1,14 @@ +--- +title: Node RAID Degraded +weight: 20 +--- + # NodeRAIDDegraded ## Meaning +RAID Array is degraded. + This alert is triggered when a node has a storage configuration with RAID array, and the array is reporting as being in a degraded state due to one or more disk failures. @@ -17,7 +24,7 @@ You can open a shell on the node and use the standard Linux utilities to diagnose the issue, but you may need to install additional software in the debug container: -```console +```shell $ NODE_NAME='' $ oc debug "node/$NODE_NAME" @@ -26,6 +33,8 @@ $ cat /proc/mdstat ## Mitigation +Cordon and drain node if possible, proceed to RAID recovery. + See the Red Hat Enterprise Linux [documentation][1] for potential steps. -[1]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/managing_storage_devices/managing-raid_managing-storage-devices +- [1](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/managing_storage_devices/managing-raid_managing-storage-devices) diff --git a/content/runbooks/node/NodeRAIDDiskFailure.md b/content/runbooks/node/NodeRAIDDiskFailure.md new file mode 100644 index 0000000..c2d9c83 --- /dev/null +++ b/content/runbooks/node/NodeRAIDDiskFailure.md @@ -0,0 +1,8 @@ +--- +title: Node RAID Disk Failure +weight: 20 +--- + +# NodeRAIDDiskFailure + +See [Node RAID Degraded]({{< ref "./NodeRAIDDegraded.md" >}}) diff --git a/content/runbooks/node/NodeTextFileCollectorScrapeError.md b/content/runbooks/node/NodeTextFileCollectorScrapeError.md new file mode 100644 index 0000000..798686b --- /dev/null +++ b/content/runbooks/node/NodeTextFileCollectorScrapeError.md @@ -0,0 +1,22 @@ +--- +title: Node Text File Collector Scrape Error +weight: 20 +--- + +# NodeTextFileCollectorScrapeError + +## Meaning + +Node Exporter text file collector failed to scrape. + +## Impact + +TODO + +## Diagnosis + +TODO + +## Mitigation + +Check if provided configuration is valid, if files were not renamed during upgrades. From ec978c7dfa1e9d541bcc0f0370ded867b739953e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Socho=C5=84?= Date: Wed, 23 Feb 2022 21:32:04 +0100 Subject: [PATCH 2/9] Update content/runbooks/node/NodeTextFileCollectorScrapeError.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Krupa --- content/runbooks/node/NodeTextFileCollectorScrapeError.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/runbooks/node/NodeTextFileCollectorScrapeError.md b/content/runbooks/node/NodeTextFileCollectorScrapeError.md index 798686b..48e1ebb 100644 --- a/content/runbooks/node/NodeTextFileCollectorScrapeError.md +++ b/content/runbooks/node/NodeTextFileCollectorScrapeError.md @@ -11,7 +11,7 @@ Node Exporter text file collector failed to scrape. ## Impact -TODO +Missing metrics from additional scripts. ## Diagnosis From 56aac1d23921b7771560fd3ca60dd02bf8a15d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Socho=C5=84?= Date: Wed, 23 Feb 2022 21:32:17 +0100 Subject: [PATCH 3/9] Update content/runbooks/node/NodeTextFileCollectorScrapeError.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Krupa --- content/runbooks/node/NodeTextFileCollectorScrapeError.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/content/runbooks/node/NodeTextFileCollectorScrapeError.md b/content/runbooks/node/NodeTextFileCollectorScrapeError.md index 48e1ebb..78258c9 100644 --- a/content/runbooks/node/NodeTextFileCollectorScrapeError.md +++ b/content/runbooks/node/NodeTextFileCollectorScrapeError.md @@ -15,7 +15,8 @@ Missing metrics from additional scripts. ## Diagnosis -TODO +- Check node_exporter logs +- Check script supervisor (like systemd or cron) for more information about failed script execution ## Mitigation From 9837103d213bab1cb9af47ed8c12a780e1f00e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Socho=C5=84?= Date: Wed, 23 Feb 2022 21:33:43 +0100 Subject: [PATCH 4/9] Update content/runbooks/node/NodeHighNumberConntrackEntriesUsed.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Krupa --- content/runbooks/node/NodeHighNumberConntrackEntriesUsed.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/content/runbooks/node/NodeHighNumberConntrackEntriesUsed.md b/content/runbooks/node/NodeHighNumberConntrackEntriesUsed.md index a672a65..4166f76 100644 --- a/content/runbooks/node/NodeHighNumberConntrackEntriesUsed.md +++ b/content/runbooks/node/NodeHighNumberConntrackEntriesUsed.md @@ -21,5 +21,4 @@ Check which apps are generating a lot of connections. ## Mitigation Migrate some pods to another nodes. -Bump conntrack limit directly on the node. -Update provisioning scripts to do the same on node start. +Bump conntrack limit directly on the node, remembering to make it persistent across node reboots. From 619d997af7488a95db02e7cfe111f879bfe23f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Socho=C5=84?= Date: Wed, 23 Feb 2022 21:33:54 +0100 Subject: [PATCH 5/9] Update content/runbooks/node/NodeClockNotSynchronising.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Krupa --- content/runbooks/node/NodeClockNotSynchronising.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/runbooks/node/NodeClockNotSynchronising.md b/content/runbooks/node/NodeClockNotSynchronising.md index a345b66..c183646 100644 --- a/content/runbooks/node/NodeClockNotSynchronising.md +++ b/content/runbooks/node/NodeClockNotSynchronising.md @@ -11,7 +11,7 @@ Clock not synchronising. ## Impact -Node instability, apps disconencting. +Time is not automatically synchronizing on the node. This can cause issues with handling TLS as well as problems with other time-sensitive applications. ## Diagnosis From a01fdf3e37472be75a58deeb4d7dbbd8daf6111b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Socho=C5=84?= Date: Wed, 23 Feb 2022 21:34:00 +0100 Subject: [PATCH 6/9] Update content/runbooks/node/NodeClockSkewDetected.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Krupa --- content/runbooks/node/NodeClockSkewDetected.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/runbooks/node/NodeClockSkewDetected.md b/content/runbooks/node/NodeClockSkewDetected.md index d9870fa..b2493dc 100644 --- a/content/runbooks/node/NodeClockSkewDetected.md +++ b/content/runbooks/node/NodeClockSkewDetected.md @@ -11,7 +11,7 @@ Clock skew detected. ## Impact -Node instability, apps disconencting. +Time is skewed on the node. This can cause issues with handling TLS as well as problems with other time-sensitive applications. ## Diagnosis From 3f9dff61decd2654e0967cf6c47b0a4940226203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Socho=C5=84?= Date: Wed, 23 Feb 2022 21:34:06 +0100 Subject: [PATCH 7/9] Update content/runbooks/node/NodeFilesystemFilesFillingUp.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Krupa --- content/runbooks/node/NodeFilesystemFilesFillingUp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/runbooks/node/NodeFilesystemFilesFillingUp.md b/content/runbooks/node/NodeFilesystemFilesFillingUp.md index ef09627..834a655 100644 --- a/content/runbooks/node/NodeFilesystemFilesFillingUp.md +++ b/content/runbooks/node/NodeFilesystemFilesFillingUp.md @@ -55,4 +55,4 @@ size. You may be able to solve the problem, or buy time, by increasing size of the storage volume. Otherwise, determine the application that is creating large numbers of files and adjust its configuration or provide it dedicated storage. -See [Node Filesystem FilesFilling Up]({{< ref "./NodeFilesystemFilesFillingUp.md" >}}) +See [Node Filesystem FilesFilling Up]({{< ref "./NodeFilesystemFilesFillingUp.md" >}}) for additional mitigation steps. From 88132acb44ceb8f84eb3c4a64a344c100ae01843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Socho=C5=84?= Date: Wed, 23 Feb 2022 21:34:13 +0100 Subject: [PATCH 8/9] Update content/runbooks/node/NodeClockNotSynchronising.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Krupa --- content/runbooks/node/NodeClockNotSynchronising.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/runbooks/node/NodeClockNotSynchronising.md b/content/runbooks/node/NodeClockNotSynchronising.md index c183646..8370926 100644 --- a/content/runbooks/node/NodeClockNotSynchronising.md +++ b/content/runbooks/node/NodeClockNotSynchronising.md @@ -19,4 +19,4 @@ TODO ## Mitigation -See [Node Clok Skew Detected]({{< ref "./NodeClockSkewDetected.md" >}}) +See [Node Clok Skew Detected]({{< ref "./NodeClockSkewDetected.md" >}}) for mitigation steps. From 441e935841ec3df1a4cb41652192005395cc3077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Socho=C5=84?= Date: Wed, 23 Feb 2022 21:54:19 +0100 Subject: [PATCH 9/9] Extend mitigation for node network*errors --- content/runbooks/node/NodeNetworkReceiveErrs.md | 10 +++++++++- content/runbooks/node/NodeNetworkTransmitErrs.md | 12 +++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/content/runbooks/node/NodeNetworkReceiveErrs.md b/content/runbooks/node/NodeNetworkReceiveErrs.md index 7c6e543..2f45588 100644 --- a/content/runbooks/node/NodeNetworkReceiveErrs.md +++ b/content/runbooks/node/NodeNetworkReceiveErrs.md @@ -21,4 +21,12 @@ Check physical cables, check networking firewall rules and so on. ## Mitigation -Cordon and drain node to migrate apps from it. +In general mitigation landscape is quite vast, some suggestions: + +- Ensure some node capacity is left unallocated (cpu/memory) for handling +networking. +- [Increase TX queue length](https://access.redhat.com/documentation/en-us/red_hat_openstack_platform/13/html/ovs-dpdk_end_to_end_troubleshooting_guide/high_packet_loss_in_the_tx_queue_of_the_instance_s_tap_interface) +- Spread services to other nodes/pods. +- Replace physical cables, change ports. +- Look into introducting Quality of Service or other +[TCP congestion avoidance algorithms](https://en.wikipedia.org/wiki/TCP_congestion_control) diff --git a/content/runbooks/node/NodeNetworkTransmitErrs.md b/content/runbooks/node/NodeNetworkTransmitErrs.md index d9e182f..dfcec44 100644 --- a/content/runbooks/node/NodeNetworkTransmitErrs.md +++ b/content/runbooks/node/NodeNetworkTransmitErrs.md @@ -17,8 +17,18 @@ Network attached storage performance issues or even data loss. ## Diagnosis Investigate networkng issues on the node and to connected hardware. +Check network interface saturation. +Check CPU usage saturation. Check physical cables, check networking firewall rules and so on. ## Mitigation -Cordon and drain node to migrate apps from it. +In general mitigation landscape is quite vast, some suggestions: + +- Ensure some node capacity is left unallocated (cpu/memory) for handling +networking. +- [Increase TX queue length](https://access.redhat.com/documentation/en-us/red_hat_openstack_platform/13/html/ovs-dpdk_end_to_end_troubleshooting_guide/high_packet_loss_in_the_tx_queue_of_the_instance_s_tap_interface) +- Spread services to other nodes/pods. +- Replace physical cables, change ports. +- Look into introducting Quality of Service or other +[TCP congestion avoidance algorithms](https://en.wikipedia.org/wiki/TCP_congestion_control)