mirror of
https://github.com/helm/charts.git
synced 2026-08-23 22:37:45 +00:00
Sync etc and general rules with coreos/prometheus-operator (#9103)
* sync etc and general rules with coreos/prometheus-operator Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com> * fix namespaces and jobs labels, fix #9034 Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com> * fix naming, bump version Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com> * revert identation to original Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com> * bump version Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com> * disable kubeEtcd.enable switch for consistency with rest of the code Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com> * sync etcd rules with etcd-io upstream Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com> * add proper variables escaping Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com> * bump version Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>
This commit is contained in:
Dmitry Verkhoturov
committed by
k8s-ci-robot
k8s-ci-robot
parent
75b020f7e8
commit
ea6bcddea1
@@ -9,7 +9,7 @@ name: prometheus-operator
|
||||
sources:
|
||||
- https://github.com/coreos/prometheus-operator
|
||||
- https://coreos.com/operators/prometheus
|
||||
version: 0.1.17
|
||||
version: 0.1.18
|
||||
appVersion: "0.25.0"
|
||||
home: https://github.com/coreos/prometheus-operator
|
||||
keywords:
|
||||
|
||||
+30
-120
@@ -692,7 +692,7 @@ spec:
|
||||
severity: warning
|
||||
- alert: KubeCronJobRunning
|
||||
annotations:
|
||||
message: CronJob {{`{{ $labels.namespaces }}`}}/{{`{{ $labels.cronjob }}`}} is taking
|
||||
message: CronJob {{`{{ $labels.namespace }}`}}/{{`{{ $labels.cronjob }}`}} is taking
|
||||
more than 1h to complete.
|
||||
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning
|
||||
expr: |
|
||||
@@ -702,7 +702,7 @@ spec:
|
||||
severity: warning
|
||||
- alert: KubeJobCompletion
|
||||
annotations:
|
||||
message: Job {{`{{ $labels.namespaces }}`}}/{{`{{ $labels.job }}`}} is taking more than
|
||||
message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} is taking more than
|
||||
one hour to complete.
|
||||
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion
|
||||
expr: |
|
||||
@@ -712,7 +712,7 @@ spec:
|
||||
severity: warning
|
||||
- alert: KubeJobFailed
|
||||
annotations:
|
||||
message: Job {{`{{ $labels.namespaces }}`}}/{{`{{ $labels.job }}`}} failed to complete.
|
||||
message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} failed to complete.
|
||||
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed
|
||||
expr: |
|
||||
kube_job_status_failed{job="kube-state-metrics"} > 0
|
||||
@@ -950,19 +950,41 @@ spec:
|
||||
- name: general.rules
|
||||
rules:
|
||||
- alert: TargetDown
|
||||
annotations:
|
||||
message: '{{`{{ $value }}`}}% of the {{`{{ $labels.job }}`}} targets are down.'
|
||||
expr: 100 * (count(up == 0) BY (job) / count(up) BY (job)) > 10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
- alert: DeadMansSwitch
|
||||
annotations:
|
||||
message: This is a DeadMansSwitch meant to ensure that the entire alerting
|
||||
pipeline is functional.
|
||||
description: '{{`{{ $value }}`}}% of {{`{{ $labels.job }}`}} targets are down.'
|
||||
summary: Targets are down
|
||||
- alert: DeadMansSwitch
|
||||
expr: vector(1)
|
||||
labels:
|
||||
severity: none
|
||||
annotations:
|
||||
description: This is a DeadMansSwitch meant to ensure that the entire Alerting
|
||||
pipeline is functional.
|
||||
summary: Alerting DeadMansSwitch
|
||||
- record: fd_utilization
|
||||
expr: process_open_fds / process_max_fds
|
||||
- alert: FdExhaustionClose
|
||||
expr: predict_linear(fd_utilization[1h], 3600 * 4) > 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: '{{`{{ $labels.job }}`}}: {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} instance
|
||||
will exhaust in file/socket descriptors within the next 4 hours'
|
||||
summary: file descriptors soon exhausted
|
||||
- alert: FdExhaustionClose
|
||||
expr: predict_linear(fd_utilization[10m], 3600) > 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: '{{`{{ $labels.job }}`}}: {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} instance
|
||||
will exhaust in file/socket descriptors within the next hour'
|
||||
summary: file descriptors soon exhausted
|
||||
- name: kube-prometheus-node-alerting.rules
|
||||
rules:
|
||||
- alert: NodeDiskRunningFull
|
||||
@@ -1111,116 +1133,4 @@ spec:
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
- name: etcd3_alert.rules
|
||||
rules:
|
||||
- alert: InsufficientMembers
|
||||
expr: count(up{job="kube-etcd"} == 0) > (count(up{job="kube-etcd"}) / 2 - 1)
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: If one more etcd member goes down the cluster will be unavailable
|
||||
summary: etcd cluster insufficient members
|
||||
- alert: NoLeader
|
||||
expr: etcd_server_has_leader{job="kube-etcd"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: etcd member {{`{{ $labels.instance }}`}} has no leader
|
||||
summary: etcd member has no leader
|
||||
- alert: HighNumberOfLeaderChanges
|
||||
expr: increase(etcd_server_leader_changes_seen_total{job="kube-etcd"}[1h]) > 3
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: etcd instance {{`{{ $labels.instance }}`}} has seen {{`{{ $value }}`}} leader
|
||||
changes within the last hour
|
||||
summary: a high number of leader changes within the etcd cluster are happening
|
||||
- alert: HighNumberOfFailedGRPCRequests
|
||||
expr: 100 * (sum(rate(grpc_server_handled_total{grpc_code!="OK",job="kube-etcd"}[5m])) BY (grpc_service, grpc_method)
|
||||
/ sum(rate(grpc_server_handled_total{job="kube-etcd"}[5m])) BY (grpc_service, grpc_method)) > 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.grpc_method }}`}} failed
|
||||
on etcd instance {{`{{ $labels.instance }}`}}'
|
||||
summary: a high number of gRPC requests are failing
|
||||
- alert: HighNumberOfFailedGRPCRequests
|
||||
expr: 100 * (sum(rate(grpc_server_handled_total{grpc_code!="OK",job="kube-etcd"}[5m])) BY (grpc_service, grpc_method)
|
||||
/ sum(rate(grpc_server_handled_total{job="kube-etcd"}[5m])) BY (grpc_service, grpc_method)) > 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.grpc_method }}`}} failed
|
||||
on etcd instance {{`{{ $labels.instance }}`}}'
|
||||
summary: a high number of gRPC requests are failing
|
||||
- alert: GRPCRequestsSlow
|
||||
expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="kube-etcd",grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le))
|
||||
> 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: on etcd instance {{`{{ $labels.instance }}`}} gRPC requests to {{`{{ $labels.grpc_method
|
||||
}}`}} are slow
|
||||
summary: slow gRPC requests
|
||||
- record: instance:fd_utilization
|
||||
expr: process_open_fds / process_max_fds
|
||||
- alert: FdExhaustionClose
|
||||
expr: predict_linear(instance:fd_utilization[1h], 3600 * 4) > 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: '{{`{{ $labels.job }}`}} instance {{`{{ $labels.instance }}`}} will exhaust
|
||||
its file descriptors soon'
|
||||
summary: file descriptors soon exhausted
|
||||
- alert: FdExhaustionClose
|
||||
expr: predict_linear(instance:fd_utilization[10m], 3600) > 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
description: '{{`{{ $labels.job }}`}} instance {{`{{ $labels.instance }}`}} will exhaust
|
||||
its file descriptors soon'
|
||||
summary: file descriptors soon exhausted
|
||||
- alert: EtcdMemberCommunicationSlow
|
||||
expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m]))
|
||||
> 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: etcd instance {{`{{ $labels.instance }}`}} member communication with
|
||||
{{`{{ $labels.To }}`}} is slow
|
||||
summary: etcd member communication is slow
|
||||
- alert: HighNumberOfFailedProposals
|
||||
expr: increase(etcd_server_proposals_failed_total{job="kube-etcd"}[1h]) > 5
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: etcd instance {{`{{ $labels.instance }}`}} has seen {{`{{ $value }}`}} proposal
|
||||
failures within the last hour
|
||||
summary: a high number of proposals within the etcd cluster are failing
|
||||
- alert: HighFsyncDurations
|
||||
expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m]))
|
||||
> 0.5
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: etcd instance {{`{{ $labels.instance }}`}} fync durations are high
|
||||
summary: high fsync durations
|
||||
- alert: HighCommitDurations
|
||||
expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m]))
|
||||
> 0.25
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: etcd instance {{`{{ $labels.instance }}`}} commit durations are high
|
||||
summary: high commit durations
|
||||
{{- end }}
|
||||
@@ -0,0 +1,144 @@
|
||||
{{- if and .Values.defaultRules.create }}
|
||||
# these rules synced manually from https://github.com/etcd-io/etcd/blob/master/Documentation/etcd-mixin/mixin.libsonnet
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: {{ printf "%s-%s" (include "prometheus-operator.fullname" .) "kube-etcd-rules" | trunc 63 | trimSuffix "-" }}
|
||||
labels:
|
||||
app: {{ template "prometheus-operator.name" . }}
|
||||
{{ include "prometheus-operator.labels" . | indent 4 }}
|
||||
spec:
|
||||
groups:
|
||||
- name: etcd
|
||||
rules:
|
||||
- alert: EtcdInsufficientMembers
|
||||
annotations:
|
||||
message: 'Etcd cluster "{{`{{ $labels.job }}`}}": insufficient members ({{`{{ $value
|
||||
}}`}}).'
|
||||
expr: |
|
||||
sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2)
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
- alert: EtcdNoLeader
|
||||
annotations:
|
||||
message: 'Etcd cluster "{{`{{ $labels.job }}`}}": member {{`{{ $labels.instance }}`}} has
|
||||
no leader.'
|
||||
expr: |
|
||||
etcd_server_has_leader{job=~".*etcd.*"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
- alert: EtcdHighNumberOfLeaderChanges
|
||||
annotations:
|
||||
message: 'Etcd cluster "{{`{{ $labels.job }}`}}": instance {{`{{ $labels.instance }}`}}
|
||||
has seen {{`{{ $value }}`}} leader changes within the last hour.'
|
||||
expr: |
|
||||
rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
- alert: EtcdHighNumberOfFailedGRPCRequests
|
||||
annotations:
|
||||
message: 'Etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}}% of requests for {{`{{
|
||||
$labels.grpc_method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}.'
|
||||
expr: |
|
||||
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
/
|
||||
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
> 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
- alert: EtcdHighNumberOfFailedGRPCRequests
|
||||
annotations:
|
||||
message: 'Etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}}% of requests for {{`{{
|
||||
$labels.grpc_method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}.'
|
||||
expr: |
|
||||
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
/
|
||||
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
> 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
- alert: EtcdGRPCRequestsSlow
|
||||
annotations:
|
||||
message: 'Etcd cluster "{{`{{ $labels.job }}`}}": gRPC requests to {{`{{ $labels.grpc_method
|
||||
}}`}} are taking {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
|
||||
expr: |
|
||||
histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
|
||||
> 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
- alert: EtcdMemberCommunicationSlow
|
||||
annotations:
|
||||
message: 'Etcd cluster "{{`{{ $labels.job }}`}}": member communication with {{`{{ $labels.To
|
||||
}}`}} is taking {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
|
||||
expr: |
|
||||
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
|
||||
> 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
- alert: EtcdHighNumberOfFailedProposals
|
||||
annotations:
|
||||
message: 'Etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}} proposal failures within
|
||||
the last hour on etcd instance {{`{{ $labels.instance }}`}}.'
|
||||
expr: |
|
||||
rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
- alert: EtcdHighFsyncDurations
|
||||
annotations:
|
||||
message: 'Etcd cluster "{{`{{ $labels.job }}`}}": 99th percentile fync durations are
|
||||
{{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
|
||||
expr: |
|
||||
histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
|
||||
> 0.5
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
- alert: EtcdHighCommitDurations
|
||||
annotations:
|
||||
message: 'Etcd cluster "{{`{{ $labels.job }}`}}": 99th percentile commit durations
|
||||
{{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
|
||||
expr: |
|
||||
histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
|
||||
> 0.25
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
- alert: EtcdHighNumberOfFailedHTTPRequests
|
||||
annotations:
|
||||
message: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.method }}`}} failed on etcd
|
||||
instance {{`{{ $labels.instance }}`}}'
|
||||
expr: |
|
||||
sum(rate(etcd_http_failed_total{job=~".*etcd.*"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
|
||||
BY (method) > 0.01
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
- alert: EtcdHighNumberOfFailedHTTPRequests
|
||||
annotations:
|
||||
message: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.method }}`}} failed on etcd
|
||||
instance {{`{{ $labels.instance }}`}}.'
|
||||
expr: |
|
||||
sum(rate(etcd_http_failed_total{job=~".*etcd.*"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
|
||||
BY (method) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
- alert: EtcdHTTPRequestsSlow
|
||||
annotations:
|
||||
message: Etcd instance {{`{{ $labels.instance }}`}} HTTP requests to {{`{{ $labels.method
|
||||
}}`}} are slow.
|
||||
expr: |
|
||||
histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
|
||||
> 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
{{- end }}
|
||||
Reference in New Issue
Block a user