diff --git a/stable/prometheus-operator/Chart.yaml b/stable/prometheus-operator/Chart.yaml index f085033d23..39c6f38c6b 100644 --- a/stable/prometheus-operator/Chart.yaml +++ b/stable/prometheus-operator/Chart.yaml @@ -12,7 +12,7 @@ sources: - https://github.com/coreos/kube-prometheus - https://github.com/coreos/prometheus-operator - https://coreos.com/operators/prometheus -version: 8.5.6 +version: 8.5.7 appVersion: 0.34.0 tillerVersion: ">=2.12.0" home: https://github.com/coreos/prometheus-operator diff --git a/stable/prometheus-operator/README.md b/stable/prometheus-operator/README.md index 2ddda24089..b65fd255d3 100644 --- a/stable/prometheus-operator/README.md +++ b/stable/prometheus-operator/README.md @@ -140,6 +140,7 @@ The following tables list the configurable parameters of the prometheus-operator | `defaultRules.annotations` | Annotations for default rules for monitoring the cluster | `{}` | | `defaultRules.create` | Create default rules for monitoring the cluster | `true` | | `defaultRules.labels` | Labels for default rules for monitoring the cluster | `{}` | +| `defaultRules.runbookUrl` | URL prefix for default rule runbook_url annotations | `https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#` | | `defaultRules.rules.PrometheusOperator` | Create Prometheus Operator default rules| `true` | | `defaultRules.rules.alertmanager` | Create default rules for Alert Manager | `true` | | `defaultRules.rules.etcd` | Create default rules for ETCD | `true` | diff --git a/stable/prometheus-operator/hack/sync_prometheus_rules.py b/stable/prometheus-operator/hack/sync_prometheus_rules.py index a59ed8bbe8..042342b110 100755 --- a/stable/prometheus-operator/hack/sync_prometheus_rules.py +++ b/stable/prometheus-operator/hack/sync_prometheus_rules.py @@ -105,6 +105,9 @@ replacement_map = { 'alertmanager-$1': { 'replacement': '$1', 'init': ''}, + 'https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#': { + 'replacement': '{{ .Values.defaultRules.runbookUrl }}', + 'init': ''}, } # standard header diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/k8s.rules.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/k8s.rules.yaml index 6ffd759462..794fc4604a 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/k8s.rules.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -22,30 +22,30 @@ spec: groups: - name: k8s.rules rules: - - expr: sum(rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container!="POD"}[5m])) by (namespace) + - expr: sum(rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m])) by (namespace) record: namespace:container_cpu_usage_seconds_total:sum_rate - expr: |- sum by (namespace, pod, container) ( - rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container!="POD"}[5m]) + rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m]) ) * on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info) record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate - expr: |- - container_memory_working_set_bytes{job="kubelet", image!=""} + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} * on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info) record: node_namespace_pod_container:container_memory_working_set_bytes - expr: |- - container_memory_rss{job="kubelet", image!=""} + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} * on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info) record: node_namespace_pod_container:container_memory_rss - expr: |- - container_memory_cache{job="kubelet", image!=""} + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} * on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info) record: node_namespace_pod_container:container_memory_cache - expr: |- - container_memory_swap{job="kubelet", image!=""} + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} * on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info) record: node_namespace_pod_container:container_memory_swap - - expr: sum(container_memory_usage_bytes{job="kubelet", image!="", container!="POD"}) by (namespace) + - expr: sum(container_memory_usage_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}) by (namespace) record: namespace:container_memory_usage_bytes:sum - expr: |- sum by (namespace) ( diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error.yaml index e5f886ea1b..baa6b79e2f 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error.yaml @@ -24,7 +24,7 @@ spec: rules: - alert: ErrorBudgetBurn annotations: - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-errorbudgetburn + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-errorbudgetburn expr: |- ( status_class_5xx:apiserver_request_total:ratio_rate1h{job="apiserver"} > (14.4*0.010000) @@ -42,7 +42,7 @@ spec: severity: critical - alert: ErrorBudgetBurn annotations: - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-errorbudgetburn + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-errorbudgetburn expr: |- ( status_class_5xx:apiserver_request_total:ratio_rate1d{job="apiserver"} > (3*0.010000) diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-absent.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-absent.yaml index 8b03919159..e94c303a0c 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-absent.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-absent.yaml @@ -30,7 +30,7 @@ spec: - alert: AlertmanagerDown annotations: message: Alertmanager has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-alertmanagerdown + runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-alertmanagerdown expr: absent(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 1) for: 15m labels: @@ -40,7 +40,7 @@ spec: - alert: CoreDNSDown annotations: message: CoreDNS has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-corednsdown + runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-corednsdown expr: absent(up{job="kube-dns"} == 1) for: 15m labels: @@ -50,7 +50,7 @@ spec: - alert: KubeAPIDown annotations: message: KubeAPI has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown + runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-kubeapidown expr: absent(up{job="apiserver"} == 1) for: 15m labels: @@ -60,7 +60,7 @@ spec: - alert: KubeControllerManagerDown annotations: message: KubeControllerManager has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown + runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-kubecontrollermanagerdown expr: absent(up{job="kube-controller-manager"} == 1) for: 15m labels: @@ -70,7 +70,7 @@ spec: - alert: KubeSchedulerDown annotations: message: KubeScheduler has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown + runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-kubeschedulerdown expr: absent(up{job="kube-scheduler"} == 1) for: 15m labels: @@ -80,7 +80,7 @@ spec: - alert: KubeStateMetricsDown annotations: message: KubeStateMetrics has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricsdown + runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-kubestatemetricsdown expr: absent(up{job="kube-state-metrics"} == 1) for: 15m labels: @@ -90,7 +90,7 @@ spec: - alert: KubeletDown annotations: message: Kubelet has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown + runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-kubeletdown expr: absent(up{job="kubelet"} == 1) for: 15m labels: @@ -100,7 +100,7 @@ spec: - alert: NodeExporterDown annotations: message: NodeExporter has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeexporterdown + runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-nodeexporterdown expr: absent(up{job="node-exporter"} == 1) for: 15m labels: @@ -109,7 +109,7 @@ spec: - alert: PrometheusDown annotations: message: Prometheus has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusdown + runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-prometheusdown expr: absent(up{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} == 1) for: 15m labels: @@ -118,10 +118,10 @@ spec: - alert: PrometheusOperatorDown annotations: message: PrometheusOperator has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusoperatordown + runbook_url: {{ .Values.defaultRules.runbook_url }}alert-name-prometheusoperatordown expr: absent(up{job="{{ $operatorJob }}",namespace="{{ $namespace }}"} == 1) for: 15m labels: severity: critical {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-apps.yaml index d334748759..6bbfd876f9 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-apps.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -25,7 +25,7 @@ spec: - alert: KubePodCrashLooping annotations: message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} ({{`{{ $labels.container }}`}}) is restarting {{`{{ printf "%.2f" $value }}`}} times / 5 minutes. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 0 for: 15m labels: @@ -33,7 +33,7 @@ spec: - alert: KubePodNotReady annotations: message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} has been in a non-ready state for longer than 15 minutes. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodnotready expr: sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"}) * on(namespace, pod) group_left(owner_kind) max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})) > 0 for: 15m labels: @@ -41,7 +41,7 @@ spec: - alert: KubeDeploymentGenerationMismatch annotations: message: Deployment generation for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.deployment }}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentgenerationmismatch expr: |- kube_deployment_status_observed_generation{job="kube-state-metrics"} != @@ -52,7 +52,7 @@ spec: - alert: KubeDeploymentReplicasMismatch annotations: message: Deployment {{`{{ $labels.namespace }}`}}/{{`{{ $labels.deployment }}`}} has not matched the expected number of replicas for longer than 15 minutes. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentreplicasmismatch expr: |- kube_deployment_spec_replicas{job="kube-state-metrics"} != @@ -63,7 +63,7 @@ spec: - alert: KubeStatefulSetReplicasMismatch annotations: message: StatefulSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} has not matched the expected number of replicas for longer than 15 minutes. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetreplicasmismatch expr: |- kube_statefulset_status_replicas_ready{job="kube-state-metrics"} != @@ -74,7 +74,7 @@ spec: - alert: KubeStatefulSetGenerationMismatch annotations: message: StatefulSet generation for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetgenerationmismatch expr: |- kube_statefulset_status_observed_generation{job="kube-state-metrics"} != @@ -85,7 +85,7 @@ spec: - alert: KubeStatefulSetUpdateNotRolledOut annotations: message: StatefulSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} update has not been rolled out. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetupdatenotrolledout expr: |- max without (revision) ( kube_statefulset_status_current_revision{job="kube-state-metrics"} @@ -104,7 +104,7 @@ spec: - alert: KubeDaemonSetRolloutStuck annotations: message: Only {{`{{ $value | humanizePercentage }}`}} of the desired Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are scheduled and ready. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetrolloutstuck expr: |- kube_daemonset_status_number_ready{job="kube-state-metrics"} / @@ -115,7 +115,7 @@ spec: - alert: KubeContainerWaiting annotations: message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} container {{`{{ $labels.container}}`}} has been in waiting state for longer than 1 hour. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontainerwaiting expr: sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0 for: 1h labels: @@ -123,7 +123,7 @@ spec: - alert: KubeDaemonSetNotScheduled annotations: message: '{{`{{ $value }}`}} Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are not scheduled.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetnotscheduled expr: |- kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} - @@ -134,7 +134,7 @@ spec: - alert: KubeDaemonSetMisScheduled annotations: message: '{{`{{ $value }}`}} Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are running where they are not supposed to run.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetmisscheduled expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0 for: 10m labels: @@ -142,7 +142,7 @@ spec: - alert: KubeCronJobRunning annotations: message: CronJob {{`{{ $labels.namespace }}`}}/{{`{{ $labels.cronjob }}`}} is taking more than 1h to complete. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecronjobrunning expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 3600 for: 1h labels: @@ -150,7 +150,7 @@ spec: - alert: KubeJobCompletion annotations: message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} is taking more than one hour to complete. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobcompletion expr: kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0 for: 1h labels: @@ -158,7 +158,7 @@ spec: - alert: KubeJobFailed annotations: message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} failed to complete. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobfailed expr: kube_job_failed{job="kube-state-metrics"} > 0 for: 15m labels: @@ -166,7 +166,7 @@ spec: - alert: KubeHpaReplicasMismatch annotations: message: HPA {{`{{ $labels.namespace }}`}}/{{`{{ $labels.hpa }}`}} has not matched the desired number of replicas for longer than 15 minutes. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubehpareplicasmismatch expr: |- (kube_hpa_status_desired_replicas{job="kube-state-metrics"} != @@ -179,7 +179,7 @@ spec: - alert: KubeHpaMaxedOut annotations: message: HPA {{`{{ $labels.namespace }}`}}/{{`{{ $labels.hpa }}`}} has been running at max replicas for longer than 15 minutes. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubehpamaxedout expr: |- kube_hpa_status_current_replicas{job="kube-state-metrics"} == diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-resources.yaml index 7093acf676..efc71a8bac 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-resources.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -25,7 +25,7 @@ spec: - alert: KubeCPUOvercommit annotations: message: Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit expr: |- sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum) / @@ -38,7 +38,7 @@ spec: - alert: KubeMemOvercommit annotations: message: Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit expr: |- sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum) / @@ -53,7 +53,7 @@ spec: - alert: KubeCPUOvercommit annotations: message: Cluster has overcommitted CPU resource requests for Namespaces. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit expr: |- sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"}) / @@ -65,7 +65,7 @@ spec: - alert: KubeMemOvercommit annotations: message: Cluster has overcommitted memory resource requests for Namespaces. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit expr: |- sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"}) / @@ -77,7 +77,7 @@ spec: - alert: KubeQuotaExceeded annotations: message: Namespace {{`{{ $labels.namespace }}`}} is using {{`{{ $value | humanizePercentage }}`}} of its {{`{{ $labels.resource }}`}} quota. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubequotaexceeded expr: |- kube_resourcequota{job="kube-state-metrics", type="used"} / ignoring(instance, job, type) @@ -89,7 +89,7 @@ spec: - alert: CPUThrottlingHigh annotations: message: '{{`{{ $value | humanizePercentage }}`}} throttling of CPU in namespace {{`{{ $labels.namespace }}`}} for container {{`{{ $labels.container }}`}} in pod {{`{{ $labels.pod }}`}}.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-cputhrottlinghigh expr: |- sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace) / diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-storage.yaml index dd12ab5427..a39832cbd4 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-storage.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -25,11 +25,11 @@ spec: - alert: KubePersistentVolumeUsageCritical annotations: message: The PersistentVolume claimed by {{`{{ $labels.persistentvolumeclaim }}`}} in Namespace {{`{{ $labels.namespace }}`}} is only {{`{{ $value | humanizePercentage }}`}} free. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeusagecritical + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeusagecritical expr: |- - kubelet_volume_stats_available_bytes{job="kubelet"} + kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"} / - kubelet_volume_stats_capacity_bytes{job="kubelet"} + kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"} < 0.03 for: 1m labels: @@ -37,22 +37,22 @@ spec: - alert: KubePersistentVolumeFullInFourDays annotations: message: Based on recent sampling, the PersistentVolume claimed by {{`{{ $labels.persistentvolumeclaim }}`}} in Namespace {{`{{ $labels.namespace }}`}} is expected to fill up within four days. Currently {{`{{ $value | humanizePercentage }}`}} is available. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumefullinfourdays expr: |- ( - kubelet_volume_stats_available_bytes{job="kubelet"} + kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"} / - kubelet_volume_stats_capacity_bytes{job="kubelet"} + kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"} ) < 0.15 and - predict_linear(kubelet_volume_stats_available_bytes{job="kubelet"}[6h], 4 * 24 * 3600) < 0 + predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 for: 1h labels: severity: critical - alert: KubePersistentVolumeErrors annotations: message: The persistent volume {{`{{ $labels.persistentvolume }}`}} has status {{`{{ $labels.phase }}`}}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeerrors expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 for: 5m labels: diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml index 3965437ea0..3d827f3a5e 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -25,7 +25,7 @@ spec: - alert: KubeAPILatencyHigh annotations: message: The API server has an abnormal latency of {{`{{ $value }}`}} seconds for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh expr: |- ( cluster:apiserver_request_duration_seconds:mean5m{job="apiserver"} @@ -48,7 +48,7 @@ spec: - alert: KubeAPILatencyHigh annotations: message: The API server has a 99th percentile latency of {{`{{ $value }}`}} seconds for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh expr: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job="apiserver",quantile="0.99"} > 4 for: 10m labels: @@ -56,7 +56,7 @@ spec: - alert: KubeAPIErrorsHigh annotations: message: API server is returning errors for {{`{{ $value | humanizePercentage }}`}} of requests. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh expr: |- sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) / @@ -67,7 +67,7 @@ spec: - alert: KubeAPIErrorsHigh annotations: message: API server is returning errors for {{`{{ $value | humanizePercentage }}`}} of requests. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh expr: |- sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) / @@ -78,7 +78,7 @@ spec: - alert: KubeAPIErrorsHigh annotations: message: API server is returning errors for {{`{{ $value | humanizePercentage }}`}} of requests for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}} {{`{{ $labels.subresource }}`}}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh expr: |- sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) by (resource,subresource,verb) / @@ -89,7 +89,7 @@ spec: - alert: KubeAPIErrorsHigh annotations: message: API server is returning errors for {{`{{ $value | humanizePercentage }}`}} of requests for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}} {{`{{ $labels.subresource }}`}}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh expr: |- sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) by (resource,subresource,verb) / @@ -100,14 +100,14 @@ spec: - alert: KubeClientCertificateExpiration annotations: message: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 labels: severity: warning - alert: KubeClientCertificateExpiration annotations: message: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 labels: severity: critical @@ -115,7 +115,7 @@ spec: - alert: KubeAPIDown annotations: message: KubeAPI has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapidown expr: absent(up{job="apiserver"} == 1) for: 15m labels: diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml index efbdef11ff..f3c41d1c0c 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -26,7 +26,7 @@ spec: - alert: KubeControllerManagerDown annotations: message: KubeControllerManager has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontrollermanagerdown expr: absent(up{job="kube-controller-manager"} == 1) for: 15m labels: diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml index 75b81c0f0f..c0b1704fa7 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -25,7 +25,7 @@ spec: - alert: KubeNodeNotReady annotations: message: '{{`{{ $labels.node }}`}} has been unready for more than 15 minutes.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodenotready expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0 for: 15m labels: @@ -33,15 +33,15 @@ spec: - alert: KubeNodeUnreachable annotations: message: '{{`{{ $labels.node }}`}} is unreachable and some workloads may be rescheduled.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeunreachable + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodeunreachable expr: kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} == 1 labels: severity: warning - alert: KubeletTooManyPods annotations: message: Kubelet '{{`{{ $labels.node }}`}}' is running at {{`{{ $value | humanizePercentage }}`}} of its Pod capacity. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods - expr: max(max(kubelet_running_pod_count{job="kubelet"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job="kubelet"}) by(node) / max(kube_node_status_capacity_pods{job="kube-state-metrics"}) by(node) > 0.95 + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubelettoomanypods + expr: max(max(kubelet_running_pod_count{job="kubelet", metrics_path="/metrics"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) by(node) / max(kube_node_status_capacity_pods{job="kube-state-metrics"}) by(node) > 0.95 for: 15m labels: severity: warning @@ -49,8 +49,8 @@ spec: - alert: KubeletDown annotations: message: Kubelet has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown - expr: absent(up{job="kubelet"} == 1) + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletdown + expr: absent(up{job="kubelet", metrics_path="/metrics"} == 1) for: 15m labels: severity: critical diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml index 72c04b3844..5f4290ef30 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -26,7 +26,7 @@ spec: - alert: KubeSchedulerDown annotations: message: KubeScheduler has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeschedulerdown expr: absent(up{job="kube-scheduler"} == 1) for: 15m labels: diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system.yaml index 74f372c2b5..dec94ef82d 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -25,7 +25,7 @@ spec: - alert: KubeVersionMismatch annotations: message: There are {{`{{ $value }}`}} different semantic versions of Kubernetes components running. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeversionmismatch expr: count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*.[0-9]*).*"))) > 1 for: 15m labels: @@ -33,7 +33,7 @@ spec: - alert: KubeClientErrors annotations: message: Kubernetes API server client '{{`{{ $labels.job }}`}}/{{`{{ $labels.instance }}`}}' is experiencing {{`{{ $value | humanizePercentage }}`}} errors.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors expr: |- (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job) / diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/node-exporter.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/node-exporter.yaml index 6c97e4ba02..f34ae761a5 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/node-exporter.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/node-exporter.yaml @@ -25,7 +25,7 @@ spec: - alert: NodeFilesystemSpaceFillingUp annotations: description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available space left and is filling up. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup summary: Filesystem is predicted to run out of space within the next 24 hours. expr: |- ( @@ -41,7 +41,7 @@ spec: - alert: NodeFilesystemSpaceFillingUp annotations: description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available space left and is filling up fast. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup summary: Filesystem is predicted to run out of space within the next 4 hours. expr: |- ( @@ -57,7 +57,7 @@ spec: - alert: NodeFilesystemAlmostOutOfSpace annotations: description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available space left. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace summary: Filesystem has less than 5% space left. expr: |- ( @@ -71,7 +71,7 @@ spec: - alert: NodeFilesystemAlmostOutOfSpace annotations: description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available space left. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace summary: Filesystem has less than 3% space left. expr: |- ( @@ -85,7 +85,7 @@ spec: - alert: NodeFilesystemFilesFillingUp annotations: description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available inodes left and is filling up. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup summary: Filesystem is predicted to run out of inodes within the next 24 hours. expr: |- ( @@ -101,7 +101,7 @@ spec: - alert: NodeFilesystemFilesFillingUp annotations: description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available inodes left and is filling up fast. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup summary: Filesystem is predicted to run out of inodes within the next 4 hours. expr: |- ( @@ -117,7 +117,7 @@ spec: - alert: NodeFilesystemAlmostOutOfFiles annotations: description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available inodes left. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles summary: Filesystem has less than 5% inodes left. expr: |- ( @@ -131,7 +131,7 @@ spec: - alert: NodeFilesystemAlmostOutOfFiles annotations: description: Filesystem on {{`{{ $labels.device }}`}} at {{`{{ $labels.instance }}`}} has only {{`{{ printf "%.2f" $value }}`}}% available inodes left. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles summary: Filesystem has less than 3% inodes left. expr: |- ( @@ -145,7 +145,7 @@ spec: - alert: NodeNetworkReceiveErrs annotations: description: '{{`{{ $labels.instance }}`}} interface {{`{{ $labels.device }}`}} has encountered {{`{{ printf "%.0f" $value }}`}} receive errors in the last two minutes.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworkreceiveerrs + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworkreceiveerrs summary: Network interface is reporting many receive errors. expr: increase(node_network_receive_errs_total[2m]) > 10 for: 1h @@ -154,7 +154,7 @@ spec: - alert: NodeNetworkTransmitErrs annotations: description: '{{`{{ $labels.instance }}`}} interface {{`{{ $labels.device }}`}} has encountered {{`{{ printf "%.0f" $value }}`}} transmit errors in the last two minutes.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworktransmiterrs + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworktransmiterrs summary: Network interface is reporting many transmit errors. expr: increase(node_network_transmit_errs_total[2m]) > 10 for: 1h diff --git a/stable/prometheus-operator/templates/prometheus/rules/kubernetes-absent.yaml b/stable/prometheus-operator/templates/prometheus/rules/kubernetes-absent.yaml index 381f4aa806..80b2b5e307 100644 --- a/stable/prometheus-operator/templates/prometheus/rules/kubernetes-absent.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules/kubernetes-absent.yaml @@ -30,7 +30,7 @@ spec: - alert: AlertmanagerDown annotations: message: Alertmanager has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-alertmanagerdown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerdown expr: absent(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 1) for: 15m labels: @@ -40,7 +40,7 @@ spec: - alert: CoreDNSDown annotations: message: CoreDNS has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-corednsdown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-corednsdown expr: absent(up{job="kube-dns"} == 1) for: 15m labels: @@ -50,7 +50,7 @@ spec: - alert: KubeAPIDown annotations: message: KubeAPI has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapidown expr: absent(up{job="apiserver"} == 1) for: 15m labels: @@ -60,7 +60,7 @@ spec: - alert: KubeControllerManagerDown annotations: message: KubeControllerManager has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontrollermanagerdown expr: absent(up{job="kube-controller-manager"} == 1) for: 15m labels: @@ -70,7 +70,7 @@ spec: - alert: KubeSchedulerDown annotations: message: KubeScheduler has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeschedulerdown expr: absent(up{job="kube-scheduler"} == 1) for: 15m labels: @@ -80,7 +80,7 @@ spec: - alert: KubeStateMetricsDown annotations: message: KubeStateMetrics has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricsdown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatemetricsdown expr: absent(up{job="kube-state-metrics"} == 1) for: 15m labels: @@ -90,7 +90,7 @@ spec: - alert: KubeletDown annotations: message: Kubelet has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletdown expr: absent(up{job="kubelet"} == 1) for: 15m labels: @@ -100,7 +100,7 @@ spec: - alert: NodeExporterDown annotations: message: NodeExporter has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeexporterdown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeexporterdown expr: absent(up{job="node-exporter"} == 1) for: 15m labels: @@ -109,7 +109,7 @@ spec: - alert: PrometheusDown annotations: message: Prometheus has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusdown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusdown expr: absent(up{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} == 1) for: 15m labels: @@ -118,7 +118,7 @@ spec: - alert: PrometheusOperatorDown annotations: message: PrometheusOperator has disappeared from Prometheus target discovery. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusoperatordown + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusoperatordown expr: absent(up{job="{{ $operatorJob }}",namespace="{{ $namespace }}"} == 1) for: 15m labels: diff --git a/stable/prometheus-operator/templates/prometheus/rules/kubernetes-apps.yaml b/stable/prometheus-operator/templates/prometheus/rules/kubernetes-apps.yaml index 0b2e541797..3dafd09a54 100644 --- a/stable/prometheus-operator/templates/prometheus/rules/kubernetes-apps.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules/kubernetes-apps.yaml @@ -25,7 +25,7 @@ spec: - alert: KubePodCrashLooping annotations: message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} ({{`{{ $labels.container }}`}}) is restarting {{`{{ printf "%.2f" $value }}`}} times / 5 minutes. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 0 for: 1h labels: @@ -33,7 +33,7 @@ spec: - alert: KubePodNotReady annotations: message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} has been in a non-ready state for longer than an hour. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodnotready expr: sum by (namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"}) > 0 for: 1h labels: @@ -41,7 +41,7 @@ spec: - alert: KubeDeploymentGenerationMismatch annotations: message: Deployment generation for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.deployment }}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentgenerationmismatch expr: |- kube_deployment_status_observed_generation{job="kube-state-metrics"} != @@ -52,7 +52,7 @@ spec: - alert: KubeDeploymentReplicasMismatch annotations: message: Deployment {{`{{ $labels.namespace }}`}}/{{`{{ $labels.deployment }}`}} has not matched the expected number of replicas for longer than an hour. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentreplicasmismatch expr: |- kube_deployment_spec_replicas{job="kube-state-metrics"} != @@ -63,7 +63,7 @@ spec: - alert: KubeStatefulSetReplicasMismatch annotations: message: StatefulSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} has not matched the expected number of replicas for longer than 15 minutes. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetreplicasmismatch expr: |- kube_statefulset_status_replicas_ready{job="kube-state-metrics"} != @@ -74,7 +74,7 @@ spec: - alert: KubeStatefulSetGenerationMismatch annotations: message: StatefulSet generation for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetgenerationmismatch expr: |- kube_statefulset_status_observed_generation{job="kube-state-metrics"} != @@ -85,7 +85,7 @@ spec: - alert: KubeStatefulSetUpdateNotRolledOut annotations: message: StatefulSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} update has not been rolled out. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetupdatenotrolledout expr: |- max without (revision) ( kube_statefulset_status_current_revision{job="kube-state-metrics"} @@ -104,7 +104,7 @@ spec: - alert: KubeDaemonSetRolloutStuck annotations: message: Only {{`{{ $value }}`}}% of the desired Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are scheduled and ready. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetrolloutstuck expr: |- kube_daemonset_status_number_ready{job="kube-state-metrics"} / @@ -115,7 +115,7 @@ spec: - alert: KubeDaemonSetNotScheduled annotations: message: '{{`{{ $value }}`}} Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are not scheduled.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetnotscheduled expr: |- kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} - @@ -126,7 +126,7 @@ spec: - alert: KubeDaemonSetMisScheduled annotations: message: '{{`{{ $value }}`}} Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are running where they are not supposed to run.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetmisscheduled expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0 for: 10m labels: @@ -134,7 +134,7 @@ spec: - alert: KubeCronJobRunning annotations: message: CronJob {{`{{ $labels.namespace }}`}}/{{`{{ $labels.cronjob }}`}} is taking more than 1h to complete. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecronjobrunning expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 3600 for: 1h labels: @@ -142,7 +142,7 @@ spec: - alert: KubeJobCompletion annotations: message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} is taking more than one hour to complete. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobcompletion expr: kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0 for: 1h labels: @@ -150,7 +150,7 @@ spec: - alert: KubeJobFailed annotations: message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} failed to complete. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobfailed expr: kube_job_status_failed{job="kube-state-metrics"} > 0 for: 1h labels: diff --git a/stable/prometheus-operator/templates/prometheus/rules/kubernetes-resources.yaml b/stable/prometheus-operator/templates/prometheus/rules/kubernetes-resources.yaml index e3709ebca2..431fc1b6a9 100644 --- a/stable/prometheus-operator/templates/prometheus/rules/kubernetes-resources.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules/kubernetes-resources.yaml @@ -25,7 +25,7 @@ spec: - alert: KubeCPUOvercommit annotations: message: Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit expr: |- sum(namespace_name:kube_pod_container_resource_requests_cpu_cores:sum) / @@ -38,7 +38,7 @@ spec: - alert: KubeMemOvercommit annotations: message: Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit expr: |- sum(namespace_name:kube_pod_container_resource_requests_memory_bytes:sum) / @@ -53,7 +53,7 @@ spec: - alert: KubeCPUOvercommit annotations: message: Cluster has overcommitted CPU resource requests for Namespaces. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit expr: |- sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"}) / @@ -65,7 +65,7 @@ spec: - alert: KubeMemOvercommit annotations: message: Cluster has overcommitted memory resource requests for Namespaces. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit expr: |- sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"}) / @@ -77,7 +77,7 @@ spec: - alert: KubeQuotaExceeded annotations: message: Namespace {{`{{ $labels.namespace }}`}} is using {{`{{ printf "%0.0f" $value }}`}}% of its {{`{{ $labels.resource }}`}} quota. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubequotaexceeded expr: |- 100 * kube_resourcequota{job="kube-state-metrics", type="used"} / ignoring(instance, job, type) @@ -89,7 +89,7 @@ spec: - alert: CPUThrottlingHigh annotations: message: '{{`{{ printf "%0.0f" $value }}`}}% throttling of CPU in namespace {{`{{ $labels.namespace }}`}} for container {{`{{ $labels.container_name }}`}} in pod {{`{{ $labels.pod_name }}`}}.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-cputhrottlinghigh expr: |- 100 * sum(increase(container_cpu_cfs_throttled_periods_total{container_name!="", }[5m])) by (container_name, pod_name, namespace) / diff --git a/stable/prometheus-operator/templates/prometheus/rules/kubernetes-storage.yaml b/stable/prometheus-operator/templates/prometheus/rules/kubernetes-storage.yaml index 18425f84ff..0cfa2bfd2b 100644 --- a/stable/prometheus-operator/templates/prometheus/rules/kubernetes-storage.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules/kubernetes-storage.yaml @@ -25,7 +25,7 @@ spec: - alert: KubePersistentVolumeUsageCritical annotations: message: The PersistentVolume claimed by {{`{{ $labels.persistentvolumeclaim }}`}} in Namespace {{`{{ $labels.namespace }}`}} is only {{`{{ printf "%0.2f" $value }}`}}% free. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeusagecritical + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeusagecritical expr: |- 100 * kubelet_volume_stats_available_bytes{job="kubelet"} / @@ -37,7 +37,7 @@ spec: - alert: KubePersistentVolumeFullInFourDays annotations: message: Based on recent sampling, the PersistentVolume claimed by {{`{{ $labels.persistentvolumeclaim }}`}} in Namespace {{`{{ $labels.namespace }}`}} is expected to fill up within four days. Currently {{`{{ printf "%0.2f" $value }}`}}% is available. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumefullinfourdays expr: |- 100 * ( kubelet_volume_stats_available_bytes{job="kubelet"} @@ -52,7 +52,7 @@ spec: - alert: KubePersistentVolumeErrors annotations: message: The persistent volume {{`{{ $labels.persistentvolume }}`}} has status {{`{{ $labels.phase }}`}}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeerrors expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 for: 5m labels: diff --git a/stable/prometheus-operator/templates/prometheus/rules/kubernetes-system.yaml b/stable/prometheus-operator/templates/prometheus/rules/kubernetes-system.yaml index 7e856fe965..00cab8e896 100644 --- a/stable/prometheus-operator/templates/prometheus/rules/kubernetes-system.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules/kubernetes-system.yaml @@ -25,7 +25,7 @@ spec: - alert: KubeNodeNotReady annotations: message: '{{`{{ $labels.node }}`}} has been unready for more than an hour.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodenotready expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0 for: 1h labels: @@ -33,7 +33,7 @@ spec: - alert: KubeVersionMismatch annotations: message: There are {{`{{ $value }}`}} different semantic versions of Kubernetes components running. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeversionmismatch expr: count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*.[0-9]*).*"))) > 1 for: 1h labels: @@ -41,7 +41,7 @@ spec: - alert: KubeClientErrors annotations: message: Kubernetes API server client '{{`{{ $labels.job }}`}}/{{`{{ $labels.instance }}`}}' is experiencing {{`{{ printf "%0.0f" $value }}`}}% errors.' - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors expr: |- (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job) / @@ -53,7 +53,7 @@ spec: - alert: KubeClientErrors annotations: message: Kubernetes API server client '{{`{{ $labels.job }}`}}/{{`{{ $labels.instance }}`}}' is experiencing {{`{{ printf "%0.0f" $value }}`}} errors / second. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors expr: sum(rate(ksm_scrape_error_total{job="kube-state-metrics"}[5m])) by (instance, job) > 0.1 for: 15m labels: @@ -61,7 +61,7 @@ spec: - alert: KubeletTooManyPods annotations: message: Kubelet {{`{{ $labels.instance }}`}} is running {{`{{ $value }}`}} Pods, close to the limit of 110. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubelettoomanypods expr: kubelet_running_pod_count{job="kubelet"} > 110 * 0.9 for: 15m labels: @@ -69,7 +69,7 @@ spec: - alert: KubeAPILatencyHigh annotations: message: The API server has a 99th percentile latency of {{`{{ $value }}`}} seconds for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh expr: cluster_quantile:apiserver_request_latencies:histogram_quantile{job="apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 1 for: 10m labels: @@ -77,7 +77,7 @@ spec: - alert: KubeAPILatencyHigh annotations: message: The API server has a 99th percentile latency of {{`{{ $value }}`}} seconds for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh expr: cluster_quantile:apiserver_request_latencies:histogram_quantile{job="apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 4 for: 10m labels: @@ -85,7 +85,7 @@ spec: - alert: KubeAPIErrorsHigh annotations: message: API server is returning errors for {{`{{ $value }}`}}% of requests. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh expr: |- sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) / @@ -96,7 +96,7 @@ spec: - alert: KubeAPIErrorsHigh annotations: message: API server is returning errors for {{`{{ $value }}`}}% of requests. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh expr: |- sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) / @@ -107,7 +107,7 @@ spec: - alert: KubeAPIErrorsHigh annotations: message: API server is returning errors for {{`{{ $value }}`}}% of requests for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}} {{`{{ $labels.subresource }}`}}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh expr: |- sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) by (resource,subresource,verb) / @@ -118,7 +118,7 @@ spec: - alert: KubeAPIErrorsHigh annotations: message: API server is returning errors for {{`{{ $value }}`}}% of requests for {{`{{ $labels.verb }}`}} {{`{{ $labels.resource }}`}} {{`{{ $labels.subresource }}`}}. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh expr: |- sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) by (resource,subresource,verb) / @@ -129,14 +129,14 @@ spec: - alert: KubeClientCertificateExpiration annotations: message: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 labels: severity: warning - alert: KubeClientCertificateExpiration annotations: message: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. - runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 labels: severity: critical diff --git a/stable/prometheus-operator/values.yaml b/stable/prometheus-operator/values.yaml index a20b60f15e..b8f3058abd 100644 --- a/stable/prometheus-operator/values.yaml +++ b/stable/prometheus-operator/values.yaml @@ -40,6 +40,9 @@ defaultRules: prometheusOperator: true time: true + ## Runbook url prefix for default rules + runbookUrl: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md# + ## Labels for default rules labels: {} ## Annotations for default rules