github/deprecated-helm-charts
1
0
Fork 0
mirror of https://github.com/helm/charts.git synced 2026-08-23 22:37:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

[stable/prometheus-operator] add runbook_url prefix (#19810) (#19811)

Browse Source 
* [stable/prometheus-operator] add runbook_url prefix (#19810)

Fixes #19810

This adds support for parametrising runbook_url configuration in the
default rules configuration and the sync script. Some minor rules updates
as the latest changes are pulled in by the sync script.

Signed-off-by: Sean Johnson <seanson@users.noreply.github.com>

* [stable/prometheus-operator] chart version bump

Signed-off-by: Sean Johnson <seanson@users.noreply.github.com>
This commit is contained in:
Sean Johnson
2020-01-13 06:47:40 -08:00
committed by Kubernetes Prow Robot
parent eedbdbb830
commit 71d5b06c15
21 changed files with 132 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
stable/prometheus-operator/Chart.yaml
+1 -1
View File
@@ -12,7 +12,7 @@ sources:
- https://github.com/coreos/kube-prometheus
- https://github.com/coreos/prometheus-operator
- https://coreos.com/operators/prometheus
version: 8.5.6
version: 8.5.7
appVersion: 0.34.0
tillerVersion: ">=2.12.0"
home: https://github.com/coreos/prometheus-operator
stable/prometheus-operator/README.md
+1
View File
@@ -140,6 +140,7 @@ The following tables list the configurable parameters of the prometheus-operator
| `defaultRules.annotations` | Annotations for default rules for monitoring the cluster | `{}` |
| `defaultRules.create` | Create default rules for monitoring the cluster | `true` |
| `defaultRules.labels` | Labels for default rules for monitoring the cluster | `{}` |
| `defaultRules.runbookUrl` | URL prefix for default rule runbook_url annotations | `https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#` |
| `defaultRules.rules.PrometheusOperator` | Create Prometheus Operator default rules| `true` |
| `defaultRules.rules.alertmanager` | Create default rules for Alert Manager | `true` |
| `defaultRules.rules.etcd` | Create default rules for ETCD | `true` |
stable/prometheus-operator/hack/sync_prometheus_rules.py
+3
View File
@@ -105,6 +105,9 @@ replacement_map = {
'alertmanager-$1': {
'replacement': '$1',
'init': ''},
'https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#': {
'replacement': '{{ .Values.defaultRules.runbookUrl }}',
'init': ''},
}
# standard header
stable/prometheus-operator/templates/prometheus/rules-1.14/k8s.rules.yaml
+7 -7
View File
@@ -22,30 +22,30 @@ spec:
groups:
- name: k8s.rules
rules:
- expr: sum(rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container!="POD"}[5m])) by (namespace)
- expr: sum(rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m])) by (namespace)
record: namespace:container_cpu_usage_seconds_total:sum_rate
- expr: |-
sum by (namespace, pod, container) (
rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container!="POD"}[5m])
rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m])
) * on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)
record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate
- expr: |-
container_memory_working_set_bytes{job="kubelet", image!=""}
container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)
record: node_namespace_pod_container:container_memory_working_set_bytes
- expr: |-
container_memory_rss{job="kubelet", image!=""}
container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)
record: node_namespace_pod_container:container_memory_rss
- expr: |-
container_memory_cache{job="kubelet", image!=""}
container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)
record: node_namespace_pod_container:container_memory_cache
- expr: |-
container_memory_swap{job="kubelet", image!=""}
container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)
record: node_namespace_pod_container:container_memory_swap
- expr: sum(container_memory_usage_bytes{job="kubelet", image!="", container!="POD"}) by (namespace)
- expr: sum(container_memory_usage_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}) by (namespace)
record: namespace:container_memory_usage_bytes:sum
- expr: |-
sum by (namespace) (
stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error.yaml
+2 -2
View File
@@ -24,7 +24,7 @@ spec:
rules:
- alert: ErrorBudgetBurn
annotations:
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-errorbudgetburn
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-errorbudgetburn
expr: |-
(
status_class_5xx:apiserver_request_total:ratio_rate1h{job="apiserver"} > (14.4*0.010000)
@@ -42,7 +42,7 @@ spec:
severity: critical
- alert: ErrorBudgetBurn
annotations:
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-errorbudgetburn
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-errorbudgetburn
expr: |-
(
status_class_5xx:apiserver_request_total:ratio_rate1d{job="apiserver"} > (3*0.010000)
stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-absent.yaml
+11 -11
View File
@@ -30,7 +30,7 @@ spec:
- alert: AlertmanagerDown
annotations:
message: Alertmanager has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-alertmanagerdown
runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-alertmanagerdown
expr: absent(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 1)
for: 15m
labels:
@@ -40,7 +40,7 @@ spec:
- alert: CoreDNSDown
annotations:
message: CoreDNS has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-corednsdown
runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-corednsdown
expr: absent(up{job="kube-dns"} == 1)
for: 15m
labels:
@@ -50,7 +50,7 @@ spec:
- alert: KubeAPIDown
annotations:
message: KubeAPI has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown
runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-kubeapidown
expr: absent(up{job="apiserver"} == 1)
for: 15m
labels:
@@ -60,7 +60,7 @@ spec:
- alert: KubeControllerManagerDown
annotations:
message: KubeControllerManager has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown
runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-kubecontrollermanagerdown
expr: absent(up{job="kube-controller-manager"} == 1)
for: 15m
labels:
@@ -70,7 +70,7 @@ spec:
- alert: KubeSchedulerDown
annotations:
message: KubeScheduler has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown
runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-kubeschedulerdown
expr: absent(up{job="kube-scheduler"} == 1)
for: 15m
labels:
@@ -80,7 +80,7 @@ spec:
- alert: KubeStateMetricsDown
annotations:
message: KubeStateMetrics has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricsdown
runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-kubestatemetricsdown
expr: absent(up{job="kube-state-metrics"} == 1)
for: 15m
labels:
@@ -90,7 +90,7 @@ spec:
- alert: KubeletDown
annotations:
message: Kubelet has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown
runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-kubeletdown
expr: absent(up{job="kubelet"} == 1)
for: 15m
labels:
@@ -100,7 +100,7 @@ spec:
- alert: NodeExporterDown
annotations:
message: NodeExporter has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeexporterdown
runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-nodeexporterdown
expr: absent(up{job="node-exporter"} == 1)
for: 15m
labels:
@@ -109,7 +109,7 @@ spec:
- alert: PrometheusDown
annotations:
message: Prometheus has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusdown
runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-prometheusdown
expr: absent(up{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} == 1)
for: 15m
labels:
@@ -118,10 +118,10 @@ spec:
- alert: PrometheusOperatorDown
annotations:
message: PrometheusOperator has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusoperatordown
runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-prometheusoperatordown
expr: absent(up{job="{{ $operatorJob }}",namespace="{{ $namespace }}"} == 1)
for: 15m
labels:
severity: critical
{{- end }}
{{- end }}
{{- end }}
stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-apps.yaml
+16 -16
View File
@@ -25,7 +25,7 @@ spec:
- alert: KubePodCrashLooping
annotations:
message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} ({{`{{ $labels.container }}`}}) is restarting {{`{{ printf "%.2f" $value }}`}} times / 5 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping
expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 0
for: 15m
labels:
@@ -33,7 +33,7 @@ spec:
- alert: KubePodNotReady
annotations:
message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} has been in a non-ready state for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodnotready
expr: sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"}) * on(namespace, pod) group_left(owner_kind) max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})) > 0
for: 15m
labels:
@@ -41,7 +41,7 @@ spec:
- alert: KubeDeploymentGenerationMismatch
annotations:
message: Deployment generation for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.deployment }}`}} does not match, this indicates that the Deployment has failed but has not been rolled back.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentgenerationmismatch
expr: |-
kube_deployment_status_observed_generation{job="kube-state-metrics"}
!=
@@ -52,7 +52,7 @@ spec:
- alert: KubeDeploymentReplicasMismatch
annotations:
message: Deployment {{`{{ $labels.namespace }}`}}/{{`{{ $labels.deployment }}`}} has not matched the expected number of replicas for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentreplicasmismatch
expr: |-
kube_deployment_spec_replicas{job="kube-state-metrics"}
!=
@@ -63,7 +63,7 @@ spec:
- alert: KubeStatefulSetReplicasMismatch
annotations:
message: StatefulSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} has not matched the expected number of replicas for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetreplicasmismatch
expr: |-
kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
!=
@@ -74,7 +74,7 @@ spec:
- alert: KubeStatefulSetGenerationMismatch
annotations:
message: StatefulSet generation for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetgenerationmismatch
expr: |-
kube_statefulset_status_observed_generation{job="kube-state-metrics"}
!=
@@ -85,7 +85,7 @@ spec:
- alert: KubeStatefulSetUpdateNotRolledOut
annotations:
message: StatefulSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} update has not been rolled out.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetupdatenotrolledout
expr: |-
max without (revision) (
kube_statefulset_status_current_revision{job="kube-state-metrics"}
@@ -104,7 +104,7 @@ spec:
- alert: KubeDaemonSetRolloutStuck
annotations:
message: Only {{`{{ $value | humanizePercentage }}`}} of the desired Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are scheduled and ready.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetrolloutstuck
expr: |-
kube_daemonset_status_number_ready{job="kube-state-metrics"}
/
@@ -115,7 +115,7 @@ spec:
- alert: KubeContainerWaiting
annotations:
message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} container {{`{{ $labels.container}}`}} has been in waiting state for longer than 1 hour.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontainerwaiting
expr: sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0
for: 1h
labels:
@@ -123,7 +123,7 @@ spec:
- alert: KubeDaemonSetNotScheduled
annotations:
message: '{{`{{ $value }}`}} Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are not scheduled.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetnotscheduled
expr: |-
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
-
@@ -134,7 +134,7 @@ spec:
- alert: KubeDaemonSetMisScheduled
annotations:
message: '{{`{{ $value }}`}} Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are running where they are not supposed to run.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetmisscheduled
expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0
for: 10m
labels:
@@ -142,7 +142,7 @@ spec:
- alert: KubeCronJobRunning
annotations:
message: CronJob {{`{{ $labels.namespace }}`}}/{{`{{ $labels.cronjob }}`}} is taking more than 1h to complete.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecronjobrunning
expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 3600
for: 1h
labels:
@@ -150,7 +150,7 @@ spec:
- alert: KubeJobCompletion
annotations:
message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} is taking more than one hour to complete.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobcompletion
expr: kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0
for: 1h
labels:
@@ -158,7 +158,7 @@ spec:
- alert: KubeJobFailed
annotations:
message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} failed to complete.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobfailed
expr: kube_job_failed{job="kube-state-metrics"} > 0
for: 15m
labels:
@@ -166,7 +166,7 @@ spec:
- alert: KubeHpaReplicasMismatch
annotations:
message: HPA {{`{{ $labels.namespace }}`}}/{{`{{ $labels.hpa }}`}} has not matched the desired number of replicas for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubehpareplicasmismatch
expr: |-
(kube_hpa_status_desired_replicas{job="kube-state-metrics"}
!=
@@ -179,7 +179,7 @@ spec:
- alert: KubeHpaMaxedOut
annotations:
message: HPA {{`{{ $labels.namespace }}`}}/{{`{{ $labels.hpa }}`}} has been running at max replicas for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubehpamaxedout
expr: |-
kube_hpa_status_current_replicas{job="kube-state-metrics"}
==
stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-resources.yaml
+6 -6
View File
@@ -25,7 +25,7 @@ spec:
- alert: KubeCPUOvercommit
annotations:
message: Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit
expr: |-
sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum)
/
@@ -38,7 +38,7 @@ spec:
- alert: KubeMemOvercommit
annotations:
message: Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit
expr: |-
sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum)
/
@@ -53,7 +53,7 @@ spec:
- alert: KubeCPUOvercommit
annotations:
message: Cluster has overcommitted CPU resource requests for Namespaces.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit
expr: |-
sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"})
/
@@ -65,7 +65,7 @@ spec:
- alert: KubeMemOvercommit
annotations:
message: Cluster has overcommitted memory resource requests for Namespaces.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit
expr: |-
sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"})
/
@@ -77,7 +77,7 @@ spec:
- alert: KubeQuotaExceeded
annotations:
message: Namespace {{`{{ $labels.namespace }}`}} is using {{`{{ $value | humanizePercentage }}`}} of its {{`{{ $labels.resource }}`}} quota.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubequotaexceeded
expr: |-
kube_resourcequota{job="kube-state-metrics", type="used"}
/ ignoring(instance, job, type)
@@ -89,7 +89,7 @@ spec:
- alert: CPUThrottlingHigh
annotations:
message: '{{`{{ $value | humanizePercentage }}`}} throttling of CPU in namespace {{`{{ $labels.namespace }}`}} for container {{`{{ $labels.container }}`}} in pod {{`{{ $labels.pod }}`}}.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-cputhrottlinghigh
expr: |-
sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace)
/
stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-storage.yaml
+8 -8
View File
@@ -25,11 +25,11 @@ spec:
- alert: KubePersistentVolumeUsageCritical
annotations:
message: The PersistentVolume claimed by {{`{{ $labels.persistentvolumeclaim }}`}} in Namespace {{`{{ $labels.namespace }}`}} is only {{`{{ $value | humanizePercentage }}`}} free.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeusagecritical
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeusagecritical
expr: |-
kubelet_volume_stats_available_bytes{job="kubelet"}
kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
/
kubelet_volume_stats_capacity_bytes{job="kubelet"}
kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
< 0.03
for: 1m
labels:
@@ -37,22 +37,22 @@ spec:
- alert: KubePersistentVolumeFullInFourDays
annotations:
message: Based on recent sampling, the PersistentVolume claimed by {{`{{ $labels.persistentvolumeclaim }}`}} in Namespace {{`{{ $labels.namespace }}`}} is expected to fill up within four days. Currently {{`{{ $value | humanizePercentage }}`}} is available.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumefullinfourdays
expr: |-
(
kubelet_volume_stats_available_bytes{job="kubelet"}
kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
/
kubelet_volume_stats_capacity_bytes{job="kubelet"}
kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
) < 0.15
and
predict_linear(kubelet_volume_stats_available_bytes{job="kubelet"}[6h], 4 * 24 * 3600) < 0
predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
for: 1h
labels:
severity: critical
- alert: KubePersistentVolumeErrors
annotations:
message: The persistent volume {{`{{ $labels.persistentvolume }}`}} has status {{`{{ $labels.phase }}`}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeerrors
expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
for: 5m
labels:
stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml
+9 -9
View File
@@ -25,7 +25,7 @@ spec:
- alert: KubeAPILatencyHigh
annotations:
message: The API server has an abnormal latency of {{`{{ $value }}`}} seconds for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh
expr: |-
(
cluster:apiserver_request_duration_seconds:mean5m{job="apiserver"}
@@ -48,7 +48,7 @@ spec:
- alert: KubeAPILatencyHigh
annotations:
message: The API server has a 99th percentile latency of {{`{{ $value }}`}} seconds for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh
expr: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job="apiserver",quantile="0.99"} > 4
for: 10m
labels:
@@ -56,7 +56,7 @@ spec:
- alert: KubeAPIErrorsHigh
annotations:
message: API server is returning errors for {{`{{ $value | humanizePercentage }}`}} of requests.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh
expr: |-
sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m]))
/
@@ -67,7 +67,7 @@ spec:
- alert: KubeAPIErrorsHigh
annotations:
message: API server is returning errors for {{`{{ $value | humanizePercentage }}`}} of requests.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh
expr: |-
sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m]))
/
@@ -78,7 +78,7 @@ spec:
- alert: KubeAPIErrorsHigh
annotations:
message: API server is returning errors for {{`{{ $value | humanizePercentage }}`}} of requests for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}} {{`{{ $labels.subresource }}`}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh
expr: |-
sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) by (resource,subresource,verb)
/
@@ -89,7 +89,7 @@ spec:
- alert: KubeAPIErrorsHigh
annotations:
message: API server is returning errors for {{`{{ $value | humanizePercentage }}`}} of requests for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}} {{`{{ $labels.subresource }}`}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh
expr: |-
sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) by (resource,subresource,verb)
/
@@ -100,14 +100,14 @@ spec:
- alert: KubeClientCertificateExpiration
annotations:
message: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration
expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800
labels:
severity: warning
- alert: KubeClientCertificateExpiration
annotations:
message: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration
expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400
labels:
severity: critical
@@ -115,7 +115,7 @@ spec:
- alert: KubeAPIDown
annotations:
message: KubeAPI has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapidown
expr: absent(up{job="apiserver"} == 1)
for: 15m
labels:
stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml
+1 -1
View File
@@ -26,7 +26,7 @@ spec:
- alert: KubeControllerManagerDown
annotations:
message: KubeControllerManager has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontrollermanagerdown
expr: absent(up{job="kube-controller-manager"} == 1)
for: 15m
labels:
stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml
+6 -6
View File
@@ -25,7 +25,7 @@ spec:
- alert: KubeNodeNotReady
annotations:
message: '{{`{{ $labels.node }}`}} has been unready for more than 15 minutes.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodenotready
expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
for: 15m
labels:
@@ -33,15 +33,15 @@ spec:
- alert: KubeNodeUnreachable
annotations:
message: '{{`{{ $labels.node }}`}} is unreachable and some workloads may be rescheduled.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeunreachable
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodeunreachable
expr: kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} == 1
labels:
severity: warning
- alert: KubeletTooManyPods
annotations:
message: Kubelet '{{`{{ $labels.node }}`}}' is running at {{`{{ $value | humanizePercentage }}`}} of its Pod capacity.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods
expr: max(max(kubelet_running_pod_count{job="kubelet"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job="kubelet"}) by(node) / max(kube_node_status_capacity_pods{job="kube-state-metrics"}) by(node) > 0.95
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubelettoomanypods
expr: max(max(kubelet_running_pod_count{job="kubelet", metrics_path="/metrics"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) by(node) / max(kube_node_status_capacity_pods{job="kube-state-metrics"}) by(node) > 0.95
for: 15m
labels:
severity: warning
@@ -49,8 +49,8 @@ spec:
- alert: KubeletDown
annotations:
message: Kubelet has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown
expr: absent(up{job="kubelet"} == 1)
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletdown
expr: absent(up{job="kubelet", metrics_path="/metrics"} == 1)
for: 15m
labels:
severity: critical
stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml
+1 -1
View File
@@ -26,7 +26,7 @@ spec:
- alert: KubeSchedulerDown
annotations:
message: KubeScheduler has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeschedulerdown
expr: absent(up{job="kube-scheduler"} == 1)
for: 15m
labels:
stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system.yaml
+2 -2
View File
@@ -25,7 +25,7 @@ spec:
- alert: KubeVersionMismatch
annotations:
message: There are {{`{{ $value }}`}} different semantic versions of Kubernetes components running.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeversionmismatch
expr: count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*.[0-9]*).*"))) > 1
for: 15m
labels:
@@ -33,7 +33,7 @@ spec:
- alert: KubeClientErrors
annotations:
message: Kubernetes API server client '{{`{{ $labels.job }}`}}/{{`{{ $labels.instance }}`}}' is experiencing {{`{{ $value | humanizePercentage }}`}} errors.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors
expr: |-
(sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job)
/
stable/prometheus-operator/templates/prometheus/rules-1.14/node-exporter.yaml
+10 -10
View File
@@ -25,7 +25,7 @@ spec:
- alert: NodeFilesystemSpaceFillingUp
annotations:
description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available space left and is filling up.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup
summary: Filesystem is predicted to run out of space within the next 24 hours.
expr: |-
(
@@ -41,7 +41,7 @@ spec:
- alert: NodeFilesystemSpaceFillingUp
annotations:
description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available space left and is filling up fast.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup
summary: Filesystem is predicted to run out of space within the next 4 hours.
expr: |-
(
@@ -57,7 +57,7 @@ spec:
- alert: NodeFilesystemAlmostOutOfSpace
annotations:
description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available space left.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace
summary: Filesystem has less than 5% space left.
expr: |-
(
@@ -71,7 +71,7 @@ spec:
- alert: NodeFilesystemAlmostOutOfSpace
annotations:
description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available space left.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace
summary: Filesystem has less than 3% space left.
expr: |-
(
@@ -85,7 +85,7 @@ spec:
- alert: NodeFilesystemFilesFillingUp
annotations:
description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available inodes left and is filling up.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup
summary: Filesystem is predicted to run out of inodes within the next 24 hours.
expr: |-
(
@@ -101,7 +101,7 @@ spec:
- alert: NodeFilesystemFilesFillingUp
annotations:
description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available inodes left and is filling up fast.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup
summary: Filesystem is predicted to run out of inodes within the next 4 hours.
expr: |-
(
@@ -117,7 +117,7 @@ spec:
- alert: NodeFilesystemAlmostOutOfFiles
annotations:
description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available inodes left.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles
summary: Filesystem has less than 5% inodes left.
expr: |-
(
@@ -131,7 +131,7 @@ spec:
- alert: NodeFilesystemAlmostOutOfFiles
annotations:
description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available inodes left.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles
summary: Filesystem has less than 3% inodes left.
expr: |-
(
@@ -145,7 +145,7 @@ spec:
- alert: NodeNetworkReceiveErrs
annotations:
description: '{{`{{ $labels.instance }}`}} interface {{`{{ $labels.device }}`}} has encountered {{`{{ printf "%.0f" $value }}`}} receive errors in the last two minutes.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworkreceiveerrs
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworkreceiveerrs
summary: Network interface is reporting many receive errors.
expr: increase(node_network_receive_errs_total[2m]) > 10
for: 1h
@@ -154,7 +154,7 @@ spec:
- alert: NodeNetworkTransmitErrs
annotations:
description: '{{`{{ $labels.instance }}`}} interface {{`{{ $labels.device }}`}} has encountered {{`{{ printf "%.0f" $value }}`}} transmit errors in the last two minutes.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworktransmiterrs
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworktransmiterrs
summary: Network interface is reporting many transmit errors.
expr: increase(node_network_transmit_errs_total[2m]) > 10
for: 1h
stable/prometheus-operator/templates/prometheus/rules/kubernetes-absent.yaml
+10 -10
View File
@@ -30,7 +30,7 @@ spec:
- alert: AlertmanagerDown
annotations:
message: Alertmanager has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-alertmanagerdown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerdown
expr: absent(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 1)
for: 15m
labels:
@@ -40,7 +40,7 @@ spec:
- alert: CoreDNSDown
annotations:
message: CoreDNS has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-corednsdown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-corednsdown
expr: absent(up{job="kube-dns"} == 1)
for: 15m
labels:
@@ -50,7 +50,7 @@ spec:
- alert: KubeAPIDown
annotations:
message: KubeAPI has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapidown
expr: absent(up{job="apiserver"} == 1)
for: 15m
labels:
@@ -60,7 +60,7 @@ spec:
- alert: KubeControllerManagerDown
annotations:
message: KubeControllerManager has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontrollermanagerdown
expr: absent(up{job="kube-controller-manager"} == 1)
for: 15m
labels:
@@ -70,7 +70,7 @@ spec:
- alert: KubeSchedulerDown
annotations:
message: KubeScheduler has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeschedulerdown
expr: absent(up{job="kube-scheduler"} == 1)
for: 15m
labels:
@@ -80,7 +80,7 @@ spec:
- alert: KubeStateMetricsDown
annotations:
message: KubeStateMetrics has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricsdown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatemetricsdown
expr: absent(up{job="kube-state-metrics"} == 1)
for: 15m
labels:
@@ -90,7 +90,7 @@ spec:
- alert: KubeletDown
annotations:
message: Kubelet has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletdown
expr: absent(up{job="kubelet"} == 1)
for: 15m
labels:
@@ -100,7 +100,7 @@ spec:
- alert: NodeExporterDown
annotations:
message: NodeExporter has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeexporterdown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeexporterdown
expr: absent(up{job="node-exporter"} == 1)
for: 15m
labels:
@@ -109,7 +109,7 @@ spec:
- alert: PrometheusDown
annotations:
message: Prometheus has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusdown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusdown
expr: absent(up{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} == 1)
for: 15m
labels:
@@ -118,7 +118,7 @@ spec:
- alert: PrometheusOperatorDown
annotations:
message: PrometheusOperator has disappeared from Prometheus target discovery.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusoperatordown
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusoperatordown
expr: absent(up{job="{{ $operatorJob }}",namespace="{{ $namespace }}"} == 1)
for: 15m
labels:
stable/prometheus-operator/templates/prometheus/rules/kubernetes-apps.yaml
+13 -13
View File
@@ -25,7 +25,7 @@ spec:
- alert: KubePodCrashLooping
annotations:
message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} ({{`{{ $labels.container }}`}}) is restarting {{`{{ printf "%.2f" $value }}`}} times / 5 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping
expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 0
for: 1h
labels:
@@ -33,7 +33,7 @@ spec:
- alert: KubePodNotReady
annotations:
message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} has been in a non-ready state for longer than an hour.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodnotready
expr: sum by (namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"}) > 0
for: 1h
labels:
@@ -41,7 +41,7 @@ spec:
- alert: KubeDeploymentGenerationMismatch
annotations:
message: Deployment generation for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.deployment }}`}} does not match, this indicates that the Deployment has failed but has not been rolled back.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentgenerationmismatch
expr: |-
kube_deployment_status_observed_generation{job="kube-state-metrics"}
!=
@@ -52,7 +52,7 @@ spec:
- alert: KubeDeploymentReplicasMismatch
annotations:
message: Deployment {{`{{ $labels.namespace }}`}}/{{`{{ $labels.deployment }}`}} has not matched the expected number of replicas for longer than an hour.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentreplicasmismatch
expr: |-
kube_deployment_spec_replicas{job="kube-state-metrics"}
!=
@@ -63,7 +63,7 @@ spec:
- alert: KubeStatefulSetReplicasMismatch
annotations:
message: StatefulSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} has not matched the expected number of replicas for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetreplicasmismatch
expr: |-
kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
!=
@@ -74,7 +74,7 @@ spec:
- alert: KubeStatefulSetGenerationMismatch
annotations:
message: StatefulSet generation for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetgenerationmismatch
expr: |-
kube_statefulset_status_observed_generation{job="kube-state-metrics"}
!=
@@ -85,7 +85,7 @@ spec:
- alert: KubeStatefulSetUpdateNotRolledOut
annotations:
message: StatefulSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} update has not been rolled out.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetupdatenotrolledout
expr: |-
max without (revision) (
kube_statefulset_status_current_revision{job="kube-state-metrics"}
@@ -104,7 +104,7 @@ spec:
- alert: KubeDaemonSetRolloutStuck
annotations:
message: Only {{`{{ $value }}`}}% of the desired Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are scheduled and ready.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetrolloutstuck
expr: |-
kube_daemonset_status_number_ready{job="kube-state-metrics"}
/
@@ -115,7 +115,7 @@ spec:
- alert: KubeDaemonSetNotScheduled
annotations:
message: '{{`{{ $value }}`}} Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are not scheduled.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetnotscheduled
expr: |-
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
-
@@ -126,7 +126,7 @@ spec:
- alert: KubeDaemonSetMisScheduled
annotations:
message: '{{`{{ $value }}`}} Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are running where they are not supposed to run.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetmisscheduled
expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0
for: 10m
labels:
@@ -134,7 +134,7 @@ spec:
- alert: KubeCronJobRunning
annotations:
message: CronJob {{`{{ $labels.namespace }}`}}/{{`{{ $labels.cronjob }}`}} is taking more than 1h to complete.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecronjobrunning
expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 3600
for: 1h
labels:
@@ -142,7 +142,7 @@ spec:
- alert: KubeJobCompletion
annotations:
message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} is taking more than one hour to complete.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobcompletion
expr: kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0
for: 1h
labels:
@@ -150,7 +150,7 @@ spec:
- alert: KubeJobFailed
annotations:
message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} failed to complete.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobfailed
expr: kube_job_status_failed{job="kube-state-metrics"} > 0
for: 1h
labels:
stable/prometheus-operator/templates/prometheus/rules/kubernetes-resources.yaml
+6 -6
View File
@@ -25,7 +25,7 @@ spec:
- alert: KubeCPUOvercommit
annotations:
message: Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit
expr: |-
sum(namespace_name:kube_pod_container_resource_requests_cpu_cores:sum)
/
@@ -38,7 +38,7 @@ spec:
- alert: KubeMemOvercommit
annotations:
message: Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit
expr: |-
sum(namespace_name:kube_pod_container_resource_requests_memory_bytes:sum)
/
@@ -53,7 +53,7 @@ spec:
- alert: KubeCPUOvercommit
annotations:
message: Cluster has overcommitted CPU resource requests for Namespaces.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit
expr: |-
sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"})
/
@@ -65,7 +65,7 @@ spec:
- alert: KubeMemOvercommit
annotations:
message: Cluster has overcommitted memory resource requests for Namespaces.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit
expr: |-
sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"})
/
@@ -77,7 +77,7 @@ spec:
- alert: KubeQuotaExceeded
annotations:
message: Namespace {{`{{ $labels.namespace }}`}} is using {{`{{ printf "%0.0f" $value }}`}}% of its {{`{{ $labels.resource }}`}} quota.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubequotaexceeded
expr: |-
100 * kube_resourcequota{job="kube-state-metrics", type="used"}
/ ignoring(instance, job, type)
@@ -89,7 +89,7 @@ spec:
- alert: CPUThrottlingHigh
annotations:
message: '{{`{{ printf "%0.0f" $value }}`}}% throttling of CPU in namespace {{`{{ $labels.namespace }}`}} for container {{`{{ $labels.container_name }}`}} in pod {{`{{ $labels.pod_name }}`}}.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-cputhrottlinghigh
expr: |-
100 * sum(increase(container_cpu_cfs_throttled_periods_total{container_name!="", }[5m])) by (container_name, pod_name, namespace)
/
stable/prometheus-operator/templates/prometheus/rules/kubernetes-storage.yaml
+3 -3
View File
@@ -25,7 +25,7 @@ spec:
- alert: KubePersistentVolumeUsageCritical
annotations:
message: The PersistentVolume claimed by {{`{{ $labels.persistentvolumeclaim }}`}} in Namespace {{`{{ $labels.namespace }}`}} is only {{`{{ printf "%0.2f" $value }}`}}% free.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeusagecritical
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeusagecritical
expr: |-
100 * kubelet_volume_stats_available_bytes{job="kubelet"}
/
@@ -37,7 +37,7 @@ spec:
- alert: KubePersistentVolumeFullInFourDays
annotations:
message: Based on recent sampling, the PersistentVolume claimed by {{`{{ $labels.persistentvolumeclaim }}`}} in Namespace {{`{{ $labels.namespace }}`}} is expected to fill up within four days. Currently {{`{{ printf "%0.2f" $value }}`}}% is available.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumefullinfourdays
expr: |-
100 * (
kubelet_volume_stats_available_bytes{job="kubelet"}
@@ -52,7 +52,7 @@ spec:
- alert: KubePersistentVolumeErrors
annotations:
message: The persistent volume {{`{{ $labels.persistentvolume }}`}} has status {{`{{ $labels.phase }}`}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeerrors
expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
for: 5m
labels:
stable/prometheus-operator/templates/prometheus/rules/kubernetes-system.yaml
+13 -13
View File
@@ -25,7 +25,7 @@ spec:
- alert: KubeNodeNotReady
annotations:
message: '{{`{{ $labels.node }}`}} has been unready for more than an hour.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodenotready
expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
for: 1h
labels:
@@ -33,7 +33,7 @@ spec:
- alert: KubeVersionMismatch
annotations:
message: There are {{`{{ $value }}`}} different semantic versions of Kubernetes components running.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeversionmismatch
expr: count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*.[0-9]*).*"))) > 1
for: 1h
labels:
@@ -41,7 +41,7 @@ spec:
- alert: KubeClientErrors
annotations:
message: Kubernetes API server client '{{`{{ $labels.job }}`}}/{{`{{ $labels.instance }}`}}' is experiencing {{`{{ printf "%0.0f" $value }}`}}% errors.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors
expr: |-
(sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job)
/
@@ -53,7 +53,7 @@ spec:
- alert: KubeClientErrors
annotations:
message: Kubernetes API server client '{{`{{ $labels.job }}`}}/{{`{{ $labels.instance }}`}}' is experiencing {{`{{ printf "%0.0f" $value }}`}} errors / second.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors
expr: sum(rate(ksm_scrape_error_total{job="kube-state-metrics"}[5m])) by (instance, job) > 0.1
for: 15m
labels:
@@ -61,7 +61,7 @@ spec:
- alert: KubeletTooManyPods
annotations:
message: Kubelet {{`{{ $labels.instance }}`}} is running {{`{{ $value }}`}} Pods, close to the limit of 110.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubelettoomanypods
expr: kubelet_running_pod_count{job="kubelet"} > 110 * 0.9
for: 15m
labels:
@@ -69,7 +69,7 @@ spec:
- alert: KubeAPILatencyHigh
annotations:
message: The API server has a 99th percentile latency of {{`{{ $value }}`}} seconds for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh
expr: cluster_quantile:apiserver_request_latencies:histogram_quantile{job="apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 1
for: 10m
labels:
@@ -77,7 +77,7 @@ spec:
- alert: KubeAPILatencyHigh
annotations:
message: The API server has a 99th percentile latency of {{`{{ $value }}`}} seconds for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh
expr: cluster_quantile:apiserver_request_latencies:histogram_quantile{job="apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 4
for: 10m
labels:
@@ -85,7 +85,7 @@ spec:
- alert: KubeAPIErrorsHigh
annotations:
message: API server is returning errors for {{`{{ $value }}`}}% of requests.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh
expr: |-
sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m]))
/
@@ -96,7 +96,7 @@ spec:
- alert: KubeAPIErrorsHigh
annotations:
message: API server is returning errors for {{`{{ $value }}`}}% of requests.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh
expr: |-
sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m]))
/
@@ -107,7 +107,7 @@ spec:
- alert: KubeAPIErrorsHigh
annotations:
message: API server is returning errors for {{`{{ $value }}`}}% of requests for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}} {{`{{ $labels.subresource }}`}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh
expr: |-
sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) by (resource,subresource,verb)
/
@@ -118,7 +118,7 @@ spec:
- alert: KubeAPIErrorsHigh
annotations:
message: API server is returning errors for {{`{{ $value }}`}}% of requests for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}} {{`{{ $labels.subresource }}`}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh
expr: |-
sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) by (resource,subresource,verb)
/
@@ -129,14 +129,14 @@ spec:
- alert: KubeClientCertificateExpiration
annotations:
message: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration
expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800
labels:
severity: warning
- alert: KubeClientCertificateExpiration
annotations:
message: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration
expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400
labels:
severity: critical
stable/prometheus-operator/values.yaml
+3
View File
@@ -40,6 +40,9 @@ defaultRules:
prometheusOperator: true
time: true
## Runbook url prefix for default rules
runbookUrl: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#
## Labels for default rules
labels: {}
## Annotations for default rules