From ea6bcddea1d025b6736ddcbbf49fd0631972f7a4 Mon Sep 17 00:00:00 2001
From: Dmitry Verkhoturov <paskal.07@gmail.com>
Date: Sun, 11 Nov 2018 21:41:39 +0300
Subject: [PATCH] Sync etc and general rules with coreos/prometheus-operator
 (#9103)

* sync etc and general rules with coreos/prometheus-operator

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>

* fix namespaces and jobs labels, fix #9034

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>

* fix naming, bump version

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>

* revert identation to original

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>

* bump version

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>

* disable kubeEtcd.enable switch for consistency with rest of the code

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>

* sync etcd rules with etcd-io upstream

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>

* add proper variables escaping

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>

* bump version

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>
---
 stable/prometheus-operator/Chart.yaml         |   2 +-
 .../rules/all.rules.yaml}                     | 150 ++++--------------
 .../alertmanager/rules/kube-etcd.rules.yaml   | 144 +++++++++++++++++
 3 files changed, 175 insertions(+), 121 deletions(-)
 rename stable/prometheus-operator/templates/{all-prometheus-rules.yaml => alertmanager/rules/all.rules.yaml} (90%)
 create mode 100644 stable/prometheus-operator/templates/alertmanager/rules/kube-etcd.rules.yaml

diff --git a/stable/prometheus-operator/Chart.yaml b/stable/prometheus-operator/Chart.yaml
index bc783615da..3df1caf5b4 100644
--- a/stable/prometheus-operator/Chart.yaml
+++ b/stable/prometheus-operator/Chart.yaml
@@ -9,7 +9,7 @@ name: prometheus-operator
 sources:
   - https://github.com/coreos/prometheus-operator
   - https://coreos.com/operators/prometheus
-version: 0.1.17
+version: 0.1.18
 appVersion: "0.25.0"
 home: https://github.com/coreos/prometheus-operator
 keywords:
diff --git a/stable/prometheus-operator/templates/all-prometheus-rules.yaml b/stable/prometheus-operator/templates/alertmanager/rules/all.rules.yaml
similarity index 90%
rename from stable/prometheus-operator/templates/all-prometheus-rules.yaml
rename to stable/prometheus-operator/templates/alertmanager/rules/all.rules.yaml
index ce6d6485a2..1b183a880c 100644
--- a/stable/prometheus-operator/templates/all-prometheus-rules.yaml
+++ b/stable/prometheus-operator/templates/alertmanager/rules/all.rules.yaml
@@ -692,7 +692,7 @@ spec:
         severity: warning
     - alert: KubeCronJobRunning
       annotations:
-        message: CronJob {{`{{ $labels.namespaces }}`}}/{{`{{ $labels.cronjob }}`}} is taking
+        message: CronJob {{`{{ $labels.namespace }}`}}/{{`{{ $labels.cronjob }}`}} is taking
           more than 1h to complete.
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning
       expr: |
@@ -702,7 +702,7 @@ spec:
         severity: warning
     - alert: KubeJobCompletion
       annotations:
-        message: Job {{`{{ $labels.namespaces }}`}}/{{`{{ $labels.job }}`}} is taking more than
+        message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} is taking more than
           one hour to complete.
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion
       expr: |
@@ -712,7 +712,7 @@ spec:
         severity: warning
     - alert: KubeJobFailed
       annotations:
-        message: Job {{`{{ $labels.namespaces }}`}}/{{`{{ $labels.job }}`}} failed to complete.
+        message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} failed to complete.
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed
       expr: |
         kube_job_status_failed{job="kube-state-metrics"}  > 0
@@ -950,19 +950,41 @@ spec:
   - name: general.rules
     rules:
     - alert: TargetDown
-      annotations:
-        message: '{{`{{ $value }}`}}% of the {{`{{ $labels.job }}`}} targets are down.'
       expr: 100 * (count(up == 0) BY (job) / count(up) BY (job)) > 10
       for: 10m
       labels:
         severity: warning
-    - alert: DeadMansSwitch
       annotations:
-        message: This is a DeadMansSwitch meant to ensure that the entire alerting
-          pipeline is functional.
+        description: '{{`{{ $value }}`}}% of {{`{{ $labels.job }}`}} targets are down.'
+        summary: Targets are down
+    - alert: DeadMansSwitch
       expr: vector(1)
       labels:
         severity: none
+      annotations:
+        description: This is a DeadMansSwitch meant to ensure that the entire Alerting
+          pipeline is functional.
+        summary: Alerting DeadMansSwitch
+    - record: fd_utilization
+      expr: process_open_fds / process_max_fds
+    - alert: FdExhaustionClose
+      expr: predict_linear(fd_utilization[1h], 3600 * 4) > 1
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        description: '{{`{{ $labels.job }}`}}: {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} instance
+          will exhaust in file/socket descriptors within the next 4 hours'
+        summary: file descriptors soon exhausted
+    - alert: FdExhaustionClose
+      expr: predict_linear(fd_utilization[10m], 3600) > 1
+      for: 10m
+      labels:
+        severity: critical
+      annotations:
+        description: '{{`{{ $labels.job }}`}}: {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} instance
+          will exhaust in file/socket descriptors within the next hour'
+        summary: file descriptors soon exhausted
   - name: kube-prometheus-node-alerting.rules
     rules:
     - alert: NodeDiskRunningFull
@@ -1111,116 +1133,4 @@ spec:
       for: 10m
       labels:
         severity: warning
-  - name: etcd3_alert.rules
-    rules:
-    - alert: InsufficientMembers
-      expr: count(up{job="kube-etcd"} == 0) > (count(up{job="kube-etcd"}) / 2 - 1)
-      for: 3m
-      labels:
-        severity: critical
-      annotations:
-        description: If one more etcd member goes down the cluster will be unavailable
-        summary: etcd cluster insufficient members
-    - alert: NoLeader
-      expr: etcd_server_has_leader{job="kube-etcd"} == 0
-      for: 1m
-      labels:
-        severity: critical
-      annotations:
-        description: etcd member {{`{{ $labels.instance }}`}} has no leader
-        summary: etcd member has no leader
-    - alert: HighNumberOfLeaderChanges
-      expr: increase(etcd_server_leader_changes_seen_total{job="kube-etcd"}[1h]) > 3
-      labels:
-        severity: warning
-      annotations:
-        description: etcd instance {{`{{ $labels.instance }}`}} has seen {{`{{ $value }}`}} leader
-          changes within the last hour
-        summary: a high number of leader changes within the etcd cluster are happening
-    - alert: HighNumberOfFailedGRPCRequests
-      expr: 100 * (sum(rate(grpc_server_handled_total{grpc_code!="OK",job="kube-etcd"}[5m])) BY (grpc_service, grpc_method)
-        / sum(rate(grpc_server_handled_total{job="kube-etcd"}[5m])) BY (grpc_service, grpc_method)) > 1
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        description: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.grpc_method }}`}} failed
-          on etcd instance {{`{{ $labels.instance }}`}}'
-        summary: a high number of gRPC requests are failing
-    - alert: HighNumberOfFailedGRPCRequests
-      expr: 100 * (sum(rate(grpc_server_handled_total{grpc_code!="OK",job="kube-etcd"}[5m])) BY (grpc_service, grpc_method)
-        / sum(rate(grpc_server_handled_total{job="kube-etcd"}[5m])) BY (grpc_service, grpc_method)) > 5
-      for: 5m
-      labels:
-        severity: critical
-      annotations:
-        description: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.grpc_method }}`}} failed
-          on etcd instance {{`{{ $labels.instance }}`}}'
-        summary: a high number of gRPC requests are failing
-    - alert: GRPCRequestsSlow
-      expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="kube-etcd",grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le))
-        > 0.15
-      for: 10m
-      labels:
-        severity: critical
-      annotations:
-        description: on etcd instance {{`{{ $labels.instance }}`}} gRPC requests to {{`{{ $labels.grpc_method
-          }}`}} are slow
-        summary: slow gRPC requests
-    - record: instance:fd_utilization
-      expr: process_open_fds / process_max_fds
-    - alert: FdExhaustionClose
-      expr: predict_linear(instance:fd_utilization[1h], 3600 * 4) > 1
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        description: '{{`{{ $labels.job }}`}} instance {{`{{ $labels.instance }}`}} will exhaust
-          its file descriptors soon'
-        summary: file descriptors soon exhausted
-    - alert: FdExhaustionClose
-      expr: predict_linear(instance:fd_utilization[10m], 3600) > 1
-      for: 10m
-      labels:
-        severity: critical
-      annotations:
-        description: '{{`{{ $labels.job }}`}} instance {{`{{ $labels.instance }}`}} will exhaust
-          its file descriptors soon'
-        summary: file descriptors soon exhausted
-    - alert: EtcdMemberCommunicationSlow
-      expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m]))
-        > 0.15
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        description: etcd instance {{`{{ $labels.instance }}`}} member communication with
-          {{`{{ $labels.To }}`}} is slow
-        summary: etcd member communication is slow
-    - alert: HighNumberOfFailedProposals
-      expr: increase(etcd_server_proposals_failed_total{job="kube-etcd"}[1h]) > 5
-      labels:
-        severity: warning
-      annotations:
-        description: etcd instance {{`{{ $labels.instance }}`}} has seen {{`{{ $value }}`}} proposal
-          failures within the last hour
-        summary: a high number of proposals within the etcd cluster are failing
-    - alert: HighFsyncDurations
-      expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m]))
-        > 0.5
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        description: etcd instance {{`{{ $labels.instance }}`}} fync durations are high
-        summary: high fsync durations
-    - alert: HighCommitDurations
-      expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m]))
-        > 0.25
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        description: etcd instance {{`{{ $labels.instance }}`}} commit durations are high
-        summary: high commit durations
 {{- end }}
diff --git a/stable/prometheus-operator/templates/alertmanager/rules/kube-etcd.rules.yaml b/stable/prometheus-operator/templates/alertmanager/rules/kube-etcd.rules.yaml
new file mode 100644
index 0000000000..19c004584d
--- /dev/null
+++ b/stable/prometheus-operator/templates/alertmanager/rules/kube-etcd.rules.yaml
@@ -0,0 +1,144 @@
+{{- if and .Values.defaultRules.create }}
+# these rules synced manually from https://github.com/etcd-io/etcd/blob/master/Documentation/etcd-mixin/mixin.libsonnet
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ printf "%s-%s" (include "prometheus-operator.fullname" .) "kube-etcd-rules" | trunc 63 | trimSuffix "-" }}
+  labels:
+    app: {{ template "prometheus-operator.name" . }}
+{{ include "prometheus-operator.labels" . | indent 4 }}
+spec:
+  groups:
+  - name: etcd
+    rules:
+    - alert: EtcdInsufficientMembers
+      annotations:
+        message: 'Etcd cluster "{{`{{ $labels.job }}`}}": insufficient members ({{`{{ $value
+          }}`}}).'
+      expr: |
+        sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2)
+      for: 3m
+      labels:
+        severity: critical
+    - alert: EtcdNoLeader
+      annotations:
+        message: 'Etcd cluster "{{`{{ $labels.job }}`}}": member {{`{{ $labels.instance }}`}} has
+          no leader.'
+      expr: |
+        etcd_server_has_leader{job=~".*etcd.*"} == 0
+      for: 1m
+      labels:
+        severity: critical
+    - alert: EtcdHighNumberOfLeaderChanges
+      annotations:
+        message: 'Etcd cluster "{{`{{ $labels.job }}`}}": instance {{`{{ $labels.instance }}`}}
+          has seen {{`{{ $value }}`}} leader changes within the last hour.'
+      expr: |
+        rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3
+      for: 15m
+      labels:
+        severity: warning
+    - alert: EtcdHighNumberOfFailedGRPCRequests
+      annotations:
+        message: 'Etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}}% of requests for {{`{{
+          $labels.grpc_method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}.'
+      expr: |
+        100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
+          /
+        sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
+          > 1
+      for: 10m
+      labels:
+        severity: warning
+    - alert: EtcdHighNumberOfFailedGRPCRequests
+      annotations:
+        message: 'Etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}}% of requests for {{`{{
+          $labels.grpc_method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}.'
+      expr: |
+        100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
+          /
+        sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
+          > 5
+      for: 5m
+      labels:
+        severity: critical
+    - alert: EtcdGRPCRequestsSlow
+      annotations:
+        message: 'Etcd cluster "{{`{{ $labels.job }}`}}": gRPC requests to {{`{{ $labels.grpc_method
+          }}`}} are taking {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
+      expr: |
+        histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
+        > 0.15
+      for: 10m
+      labels:
+        severity: critical
+    - alert: EtcdMemberCommunicationSlow
+      annotations:
+        message: 'Etcd cluster "{{`{{ $labels.job }}`}}": member communication with {{`{{ $labels.To
+          }}`}} is taking {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
+      expr: |
+        histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
+        > 0.15
+      for: 10m
+      labels:
+        severity: warning
+    - alert: EtcdHighNumberOfFailedProposals
+      annotations:
+        message: 'Etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}} proposal failures within
+          the last hour on etcd instance {{`{{ $labels.instance }}`}}.'
+      expr: |
+        rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
+      for: 15m
+      labels:
+        severity: warning
+    - alert: EtcdHighFsyncDurations
+      annotations:
+        message: 'Etcd cluster "{{`{{ $labels.job }}`}}": 99th percentile fync durations are
+          {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
+      expr: |
+        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
+        > 0.5
+      for: 10m
+      labels:
+        severity: warning
+    - alert: EtcdHighCommitDurations
+      annotations:
+        message: 'Etcd cluster "{{`{{ $labels.job }}`}}": 99th percentile commit durations
+          {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
+      expr: |
+        histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
+        > 0.25
+      for: 10m
+      labels:
+        severity: warning
+    - alert: EtcdHighNumberOfFailedHTTPRequests
+      annotations:
+        message: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.method }}`}} failed on etcd
+          instance {{`{{ $labels.instance }}`}}'
+      expr: |
+        sum(rate(etcd_http_failed_total{job=~".*etcd.*"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
+        BY (method) > 0.01
+      for: 10m
+      labels:
+        severity: warning
+    - alert: EtcdHighNumberOfFailedHTTPRequests
+      annotations:
+        message: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.method }}`}} failed on etcd
+          instance {{`{{ $labels.instance }}`}}.'
+      expr: |
+        sum(rate(etcd_http_failed_total{job=~".*etcd.*"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
+        BY (method) > 0.05
+      for: 10m
+      labels:
+        severity: critical
+    - alert: EtcdHTTPRequestsSlow
+      annotations:
+        message: Etcd instance {{`{{ $labels.instance }}`}} HTTP requests to {{`{{ $labels.method
+          }}`}} are slow.
+      expr: |
+        histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
+        > 0.15
+      for: 10m
+      labels:
+        severity: warning
+{{- end }}