From ebedffb3456a2fa8bcd927f95d9d46e0a4d2cf54 Mon Sep 17 00:00:00 2001 From: Denis Iskandarov Date: Sat, 18 Apr 2020 23:45:35 +0400 Subject: [PATCH] [stable/prometheus-operator] proper sync of rules and dashboard with upstream (#22003) * sync prometheus rules with upstream Signed-off-by: Denis Iskandarov * sync grafana dashboards with upstream Signed-off-by: Denis Iskandarov * fix link to coredns dashboard src Signed-off-by: Denis Iskandarov * chart version bump Signed-off-by: Denis Iskandarov --- stable/prometheus-operator/Chart.yaml | 2 +- stable/prometheus-operator/hack/README.md | 2 +- .../grafana/dashboards-1.14/apiserver.yaml | 4 +- .../grafana/dashboards-1.14/pods.yaml | 678 ------------------ .../prometheus-remote-write.yaml | 38 +- .../kube-apiserver-error-alerts.yaml | 65 -- .../rules-1.14/kube-apiserver-error.yaml | 139 ---- .../rules-1.14/kube-apiserver.rules.yaml | 6 +- .../rules-1.14/kubernetes-absent.yaml | 129 ---- .../kubernetes-system-apiserver.yaml | 19 - .../rules-1.14/kubernetes-system-kubelet.yaml | 2 +- .../prometheus/rules-1.14/node-time.yaml | 34 - .../prometheus/rules-1.14/prometheus.yaml | 6 +- 13 files changed, 31 insertions(+), 1093 deletions(-) delete mode 100644 stable/prometheus-operator/templates/grafana/dashboards-1.14/pods.yaml delete mode 100644 stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error-alerts.yaml delete mode 100644 stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error.yaml delete mode 100644 stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-absent.yaml delete mode 100644 stable/prometheus-operator/templates/prometheus/rules-1.14/node-time.yaml diff --git a/stable/prometheus-operator/Chart.yaml b/stable/prometheus-operator/Chart.yaml index 5e1b3eed12..9e63feb35e 100644 --- a/stable/prometheus-operator/Chart.yaml +++ b/stable/prometheus-operator/Chart.yaml @@ -12,7 +12,7 @@ sources: - https://github.com/coreos/kube-prometheus - https://github.com/coreos/prometheus-operator - https://coreos.com/operators/prometheus -version: 8.12.15 +version: 8.13.0 appVersion: 0.38.1 tillerVersion: ">=2.12.0" home: https://github.com/coreos/prometheus-operator diff --git a/stable/prometheus-operator/hack/README.md b/stable/prometheus-operator/hack/README.md index 6c164b7456..d383726d9b 100644 --- a/stable/prometheus-operator/hack/README.md +++ b/stable/prometheus-operator/hack/README.md @@ -50,4 +50,4 @@ Currently following imported: - run sync_grafana_dashboards.py inside your fork of this repo - send PR with changes to this repo -[CoreDNS dashboard](https://github.com/helm/charts/blob/master/stable/prometheus-operator/templates/grafana/dashboards/k8s-coredns.yaml) is the only dashboard which is maintained in this repo and can be changed without import. +[CoreDNS dashboard](https://github.com/helm/charts/blob/master/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-coredns.yaml) is the only dashboard which is maintained in this repo and can be changed without import. diff --git a/stable/prometheus-operator/templates/grafana/dashboards-1.14/apiserver.yaml b/stable/prometheus-operator/templates/grafana/dashboards-1.14/apiserver.yaml index d9d4c38cf3..1d67b33629 100644 --- a/stable/prometheus-operator/templates/grafana/dashboards-1.14/apiserver.yaml +++ b/stable/prometheus-operator/templates/grafana/dashboards-1.14/apiserver.yaml @@ -298,7 +298,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "apiserver_request:availability7d{verb=\"read\"}", + "expr": "apiserver_request:availability30d{verb=\"read\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -669,7 +669,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "apiserver_request:availability7d{verb=\"write\"}", + "expr": "apiserver_request:availability30d{verb=\"write\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "", diff --git a/stable/prometheus-operator/templates/grafana/dashboards-1.14/pods.yaml b/stable/prometheus-operator/templates/grafana/dashboards-1.14/pods.yaml deleted file mode 100644 index e42d5d2783..0000000000 --- a/stable/prometheus-operator/templates/grafana/dashboards-1.14/pods.yaml +++ /dev/null @@ -1,678 +0,0 @@ -{{- /* -Generated from 'pods' from https://raw.githubusercontent.com/coreos/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/helm/charts/tree/master/stable/prometheus-operator/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ $.Release.Namespace }} - name: {{ printf "%s-%s" (include "prometheus-operator.fullname" $) "pods" | trunc 63 | trimSuffix "-" }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "prometheus-operator.name" $ }}-grafana -{{ include "prometheus-operator.labels" $ | indent 4 }} -data: - pods.json: |- - { - "__inputs": [ - - ], - "__requires": [ - - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "$datasource", - "enable": true, - "expr": "time() == BOOL timestamp(rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[2m]) > 0)", - "hide": false, - "iconColor": "rgba(215, 44, 44, 1)", - "name": "Restarts", - "showIn": 0, - "tags": [ - "restart" - ], - "type": "rows" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ - - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(container) (container_memory_usage_bytes{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Current: {{`{{ container }}`}}", - "refId": "A" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Requested: {{`{{ container }}`}}", - "refId": "B" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Limit: {{`{{ container }}`}}", - "refId": "C" - }, - { - "expr": "sum by(container) (container_memory_cache{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Cache: {{`{{ container }}`}}", - "refId": "D" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (container) (irate(container_cpu_usage_seconds_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", image!=\"\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"}[4m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Current: {{`{{ container }}`}}", - "refId": "A" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Requested: {{`{{ container }}`}}", - "refId": "B" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Limit: {{`{{ container }}`}}", - "refId": "C" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sort_desc(sum by (pod) (irate(container_network_receive_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[4m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RX: {{`{{ pod }}`}}", - "refId": "A" - }, - { - "expr": "sort_desc(sum by (pod) (irate(container_network_transmit_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[4m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "TX: {{`{{ pod }}`}}", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Network I/O", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max by (container) (kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Restarts: {{`{{ container }}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Total Restarts Per Container", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Pod", - "multi": false, - "name": "pod", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=~\"$namespace\"}, pod)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": true, - "label": "Container", - "multi": false, - "name": "container", - "options": [ - - ], - "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}, container)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Pods", - "uid": "ab4f13a9892a76a4d21ce8c2445bf4ea", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/stable/prometheus-operator/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/stable/prometheus-operator/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml index 2c7d09e366..ca0d3f0019 100644 --- a/stable/prometheus-operator/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml +++ b/stable/prometheus-operator/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -88,10 +88,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(queue) group_right(instance) prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}\n)\n", + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}\n)\n", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -179,10 +179,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (queue) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n)\n", + "expr": "(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n)\n", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -283,10 +283,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(queue) group_right(instance) rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n", + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -391,7 +391,7 @@ data: "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -482,7 +482,7 @@ data: "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -573,7 +573,7 @@ data: "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -664,7 +664,7 @@ data: "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -768,7 +768,7 @@ data: "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -859,7 +859,7 @@ data: "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -1054,7 +1054,7 @@ data: "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", "refId": "A" } ], @@ -1158,7 +1158,7 @@ data: "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -1249,7 +1249,7 @@ data: "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -1340,7 +1340,7 @@ data: "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -1431,7 +1431,7 @@ data: "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}-{{`{{`}}queue{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", "refId": "A" } ], @@ -1584,11 +1584,11 @@ data: "includeAll": true, "label": null, "multi": false, - "name": "queue", + "name": "url", "options": [ ], - "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, queue)", + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", "refresh": 2, "regex": "", "sort": 0, diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error-alerts.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error-alerts.yaml deleted file mode 100644 index 3103e1c091..0000000000 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error-alerts.yaml +++ /dev/null @@ -1,65 +0,0 @@ -{{- /* -Generated from 'kube-apiserver-error-alerts' group from https://raw.githubusercontent.com/coreos/kube-prometheus/master/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/helm/charts/tree/master/stable/prometheus-operator/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "prometheus-operator.fullname" .) "kube-apiserver-error-alerts" | trunc 63 | trimSuffix "-" }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "prometheus-operator.name" . }} -{{ include "prometheus-operator.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kube-apiserver-error-alerts - rules: - - alert: ErrorBudgetBurn - annotations: - message: 'High requests error budget burn for job=apiserver (current value: {{`{{`}} $value {{`}}`}})' - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-errorbudgetburn - expr: |- - ( - status_class_5xx:apiserver_request_total:ratio_rate1h{job="apiserver"} > (14.4*0.010000) - and - status_class_5xx:apiserver_request_total:ratio_rate5m{job="apiserver"} > (14.4*0.010000) - ) - or - ( - status_class_5xx:apiserver_request_total:ratio_rate6h{job="apiserver"} > (6*0.010000) - and - status_class_5xx:apiserver_request_total:ratio_rate30m{job="apiserver"} > (6*0.010000) - ) - labels: - job: apiserver - severity: critical - - alert: ErrorBudgetBurn - annotations: - message: 'High requests error budget burn for job=apiserver (current value: {{`{{`}} $value {{`}}`}})' - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-errorbudgetburn - expr: |- - ( - status_class_5xx:apiserver_request_total:ratio_rate1d{job="apiserver"} > (3*0.010000) - and - status_class_5xx:apiserver_request_total:ratio_rate2h{job="apiserver"} > (3*0.010000) - ) - or - ( - status_class_5xx:apiserver_request_total:ratio_rate3d{job="apiserver"} > (0.010000) - and - status_class_5xx:apiserver_request_total:ratio_rate6h{job="apiserver"} > (0.010000) - ) - labels: - job: apiserver - severity: warning -{{- end }} \ No newline at end of file diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error.yaml deleted file mode 100644 index 8d998e74df..0000000000 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-error.yaml +++ /dev/null @@ -1,139 +0,0 @@ -{{- /* -Generated from 'kube-apiserver-error' group from https://raw.githubusercontent.com/coreos/kube-prometheus/master/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/helm/charts/tree/master/stable/prometheus-operator/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverError }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "prometheus-operator.fullname" .) "kube-apiserver-error" | trunc 63 | trimSuffix "-" }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "prometheus-operator.name" . }} -{{ include "prometheus-operator.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kube-apiserver-error - rules: - - expr: |- - sum by (status_class) ( - label_replace( - rate(apiserver_request_total{job="apiserver"}[5m] - ), "status_class", "${1}xx", "code", "([0-9])..") - ) - labels: - job: apiserver - record: status_class:apiserver_request_total:rate5m - - expr: |- - sum by (status_class) ( - label_replace( - rate(apiserver_request_total{job="apiserver"}[30m] - ), "status_class", "${1}xx", "code", "([0-9])..") - ) - labels: - job: apiserver - record: status_class:apiserver_request_total:rate30m - - expr: |- - sum by (status_class) ( - label_replace( - rate(apiserver_request_total{job="apiserver"}[1h] - ), "status_class", "${1}xx", "code", "([0-9])..") - ) - labels: - job: apiserver - record: status_class:apiserver_request_total:rate1h - - expr: |- - sum by (status_class) ( - label_replace( - rate(apiserver_request_total{job="apiserver"}[2h] - ), "status_class", "${1}xx", "code", "([0-9])..") - ) - labels: - job: apiserver - record: status_class:apiserver_request_total:rate2h - - expr: |- - sum by (status_class) ( - label_replace( - rate(apiserver_request_total{job="apiserver"}[6h] - ), "status_class", "${1}xx", "code", "([0-9])..") - ) - labels: - job: apiserver - record: status_class:apiserver_request_total:rate6h - - expr: |- - sum by (status_class) ( - label_replace( - rate(apiserver_request_total{job="apiserver"}[1d] - ), "status_class", "${1}xx", "code", "([0-9])..") - ) - labels: - job: apiserver - record: status_class:apiserver_request_total:rate1d - - expr: |- - sum by (status_class) ( - label_replace( - rate(apiserver_request_total{job="apiserver"}[3d] - ), "status_class", "${1}xx", "code", "([0-9])..") - ) - labels: - job: apiserver - record: status_class:apiserver_request_total:rate3d - - expr: |- - sum(status_class:apiserver_request_total:rate5m{job="apiserver",status_class="5xx"}) - / - sum(status_class:apiserver_request_total:rate5m{job="apiserver"}) - labels: - job: apiserver - record: status_class_5xx:apiserver_request_total:ratio_rate5m - - expr: |- - sum(status_class:apiserver_request_total:rate30m{job="apiserver",status_class="5xx"}) - / - sum(status_class:apiserver_request_total:rate30m{job="apiserver"}) - labels: - job: apiserver - record: status_class_5xx:apiserver_request_total:ratio_rate30m - - expr: |- - sum(status_class:apiserver_request_total:rate1h{job="apiserver",status_class="5xx"}) - / - sum(status_class:apiserver_request_total:rate1h{job="apiserver"}) - labels: - job: apiserver - record: status_class_5xx:apiserver_request_total:ratio_rate1h - - expr: |- - sum(status_class:apiserver_request_total:rate2h{job="apiserver",status_class="5xx"}) - / - sum(status_class:apiserver_request_total:rate2h{job="apiserver"}) - labels: - job: apiserver - record: status_class_5xx:apiserver_request_total:ratio_rate2h - - expr: |- - sum(status_class:apiserver_request_total:rate6h{job="apiserver",status_class="5xx"}) - / - sum(status_class:apiserver_request_total:rate6h{job="apiserver"}) - labels: - job: apiserver - record: status_class_5xx:apiserver_request_total:ratio_rate6h - - expr: |- - sum(status_class:apiserver_request_total:rate1d{job="apiserver",status_class="5xx"}) - / - sum(status_class:apiserver_request_total:rate1d{job="apiserver"}) - labels: - job: apiserver - record: status_class_5xx:apiserver_request_total:ratio_rate1d - - expr: |- - sum(status_class:apiserver_request_total:rate3d{job="apiserver",status_class="5xx"}) - / - sum(status_class:apiserver_request_total:rate3d{job="apiserver"}) - labels: - job: apiserver - record: status_class_5xx:apiserver_request_total:ratio_rate3d -{{- end }} \ No newline at end of file diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml index b5b4e0cc1f..06d6351415 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml @@ -345,11 +345,13 @@ spec: labels: verb: write record: apiserver_request:availability30d - - expr: sum by (code) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30d])) + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver"}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) labels: verb: read record: code:apiserver_request_total:increase30d - - expr: sum by (code) (increase(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30d])) + - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) labels: verb: write record: code:apiserver_request_total:increase30d diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-absent.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-absent.yaml deleted file mode 100644 index 53fb0edeae..0000000000 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-absent.yaml +++ /dev/null @@ -1,129 +0,0 @@ -{{- /* -Generated from 'kubernetes-absent' group from https://raw.githubusercontent.com/coreos/kube-prometheus/master/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/helm/charts/tree/master/stable/prometheus-operator/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesAbsent }} -{{- $operatorJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "operator" }} -{{- $prometheusJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "prometheus" }} -{{- $alertmanagerJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "alertmanager" }} -{{- $namespace := .Release.Namespace }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "prometheus-operator.fullname" .) "kubernetes-absent" | trunc 63 | trimSuffix "-" }} - namespace: {{ $namespace }} - labels: - app: {{ template "prometheus-operator.name" . }} -{{ include "prometheus-operator.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kubernetes-absent - rules: -{{- if .Values.alertmanager.enabled }} - - alert: AlertmanagerDown - annotations: - message: Alertmanager has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerdown - expr: absent(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 1) - for: 15m - labels: - severity: critical -{{- end }} -{{- if .Values.kubeDns.enabled }} - - alert: CoreDNSDown - annotations: - message: CoreDNS has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-corednsdown - expr: absent(up{job="kube-dns"} == 1) - for: 15m - labels: - severity: critical -{{- end }} -{{- if .Values.kubeApiServer.enabled }} - - alert: KubeAPIDown - annotations: - message: KubeAPI has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapidown - expr: absent(up{job="apiserver"} == 1) - for: 15m - labels: - severity: critical -{{- end }} -{{- if .Values.kubeControllerManager.enabled }} - - alert: KubeControllerManagerDown - annotations: - message: KubeControllerManager has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontrollermanagerdown - expr: absent(up{job="kube-controller-manager"} == 1) - for: 15m - labels: - severity: critical -{{- end }} -{{- if .Values.kubeScheduler.enabled }} - - alert: KubeSchedulerDown - annotations: - message: KubeScheduler has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeschedulerdown - expr: absent(up{job="kube-scheduler"} == 1) - for: 15m - labels: - severity: critical -{{- end }} -{{- if .Values.kubeStateMetrics.enabled }} - - alert: KubeStateMetricsDown - annotations: - message: KubeStateMetrics has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatemetricsdown - expr: absent(up{job="kube-state-metrics"} == 1) - for: 15m - labels: - severity: critical -{{- end }} -{{- if .Values.prometheusOperator.kubeletService.enabled }} - - alert: KubeletDown - annotations: - message: Kubelet has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletdown - expr: absent(up{job="kubelet"} == 1) - for: 15m - labels: - severity: critical -{{- end }} -{{- if .Values.nodeExporter.enabled }} - - alert: NodeExporterDown - annotations: - message: NodeExporter has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeexporterdown - expr: absent(up{job="node-exporter"} == 1) - for: 15m - labels: - severity: critical -{{- end }} - - alert: PrometheusDown - annotations: - message: Prometheus has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusdown - expr: absent(up{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.prometheusOperator.enabled }} - - alert: PrometheusOperatorDown - annotations: - message: PrometheusOperator has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusoperatordown - expr: absent(up{job="{{ $operatorJob }}",namespace="{{ $namespace }}"} == 1) - for: 15m - labels: - severity: critical -{{- end }} -{{- end }} diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml index 02301214c8..f975a51636 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -47,25 +47,6 @@ spec: for: 5m labels: severity: warning - - alert: KubeAPILatencyHigh - annotations: - message: The API server has a 99th percentile latency of {{`{{`}} $value {{`}}`}} seconds for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}}. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh - expr: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job="apiserver",quantile="0.99"} > 4 - for: 10m - labels: - severity: critical - - alert: KubeAPIErrorsHigh - annotations: - message: API server is returning errors for {{`{{`}} $value | humanizePercentage {{`}}`}} of requests for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}} {{`{{`}} $labels.subresource {{`}}`}}. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh - expr: |- - sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) by (resource,subresource,verb) - / - sum(rate(apiserver_request_total{job="apiserver"}[5m])) by (resource,subresource,verb) > 0.10 - for: 10m - labels: - severity: critical - alert: KubeAPIErrorsHigh annotations: message: API server is returning errors for {{`{{`}} $value | humanizePercentage {{`}}`}} of requests for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}} {{`{{`}} $labels.subresource {{`}}`}}. diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml index 265121f6ae..38050de6f4 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -68,7 +68,7 @@ spec: annotations: message: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletpodstartuplatencyhigh - expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name > 60 + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60 for: 15m labels: severity: warning diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/node-time.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/node-time.yaml deleted file mode 100644 index f792873476..0000000000 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/node-time.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- /* -Generated from 'node-time' group from https://raw.githubusercontent.com/coreos/kube-prometheus/master/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/helm/charts/tree/master/stable/prometheus-operator/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.time }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "prometheus-operator.fullname" .) "node-time" | trunc 63 | trimSuffix "-" }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "prometheus-operator.name" . }} -{{ include "prometheus-operator.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: node-time - rules: - - alert: ClockSkewDetected - annotations: - message: Clock skew detected on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}. Ensure NTP is configured correctly on this host. - expr: abs(node_timex_offset_seconds{job="node-exporter"}) > 0.05 - for: 2m - labels: - severity: warning -{{- end }} \ No newline at end of file diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/prometheus.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/prometheus.yaml index ec9b6bc822..bbd51e219f 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/prometheus.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/prometheus.yaml @@ -135,7 +135,7 @@ spec: severity: warning - alert: PrometheusRemoteStorageFailures annotations: - description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} if $labels.queue {{`}}`}}{{`{{`}} $labels.queue {{`}}`}}{{`{{`}} else {{`}}`}}{{`{{`}} $labels.url {{`}}`}}{{`{{`}} end {{`}}`}}. + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} summary: Prometheus fails to send samples to remote storage. expr: |- ( @@ -154,7 +154,7 @@ spec: severity: critical - alert: PrometheusRemoteWriteBehind annotations: - description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} if $labels.queue {{`}}`}}{{`{{`}} $labels.queue {{`}}`}}{{`{{`}} else {{`}}`}}{{`{{`}} $labels.url {{`}}`}}{{`{{`}} end {{`}}`}}. + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. summary: Prometheus remote write is behind. expr: |- # Without max_over_time, failed scrapes could create false negatives, see @@ -170,7 +170,7 @@ spec: severity: critical - alert: PrometheusRemoteWriteDesiredShards annotations: - description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. expr: |- # Without max_over_time, failed scrapes could create false negatives, see