From e62da84ba561e9b2cd8064834222e6ae22966ec2 Mon Sep 17 00:00:00 2001 From: Dan Sexton Date: Wed, 8 Jul 2020 06:13:08 -0400 Subject: [PATCH] [stable/prometheus-operator] Sync dashboards and rules for #22081 (#23024) Signed-off-by: Dan Sexton --- stable/prometheus-operator/Chart.yaml | 2 +- .../grafana/dashboards-1.14/apiserver.yaml | 435 ++++-------------- .../k8s-resources-cluster.yaml | 6 +- .../k8s-resources-namespace.yaml | 1 + .../dashboards-1.14/k8s-resources-node.yaml | 32 +- .../dashboards-1.14/k8s-resources-pod.yaml | 6 + .../k8s-resources-workload.yaml | 1 + .../k8s-resources-workloads-namespace.yaml | 3 +- .../rules-1.14/alertmanager.rules.yaml | 12 +- .../prometheus/rules-1.14/k8s.rules.yaml | 10 +- .../kube-apiserver-availability.rules.yaml | 146 ++++++ .../rules-1.14/kube-apiserver-slos.yaml | 8 + .../rules-1.14/kube-apiserver.rules.yaml | 86 +--- .../rules-1.14/kubernetes-apps.yaml | 59 +-- .../rules-1.14/kubernetes-resources.yaml | 8 +- .../kubernetes-system-apiserver.yaml | 34 -- .../prometheus/rules-1.14/node-exporter.yaml | 12 +- .../prometheus/rules-1.14/node.rules.yaml | 4 +- 18 files changed, 352 insertions(+), 513 deletions(-) create mode 100644 stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml diff --git a/stable/prometheus-operator/Chart.yaml b/stable/prometheus-operator/Chart.yaml index e17e50f658..ba91022f59 100644 --- a/stable/prometheus-operator/Chart.yaml +++ b/stable/prometheus-operator/Chart.yaml @@ -12,7 +12,7 @@ sources: - https://github.com/coreos/kube-prometheus - https://github.com/coreos/prometheus-operator - https://coreos.com/operators/prometheus -version: 8.15.11 +version: 8.15.12 appVersion: 0.38.1 tillerVersion: ">=2.12.0" home: https://github.com/coreos/prometheus-operator diff --git a/stable/prometheus-operator/templates/grafana/dashboards-1.14/apiserver.yaml b/stable/prometheus-operator/templates/grafana/dashboards-1.14/apiserver.yaml index 627def907d..c60b0acb0e 100644 --- a/stable/prometheus-operator/templates/grafana/dashboards-1.14/apiserver.yaml +++ b/stable/prometheus-operator/templates/grafana/dashboards-1.14/apiserver.yaml @@ -37,6 +37,24 @@ data: "id": null, "links": [ + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } ], "refresh": "10s", "rows": [ @@ -55,6 +73,7 @@ data: ], "datasource": "$datasource", "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", "format": "percentunit", "gauge": { "maxValue": 100, @@ -66,7 +85,7 @@ data: "gridPos": { }, - "id": 2, + "id": 3, "interval": null, "links": [ @@ -114,7 +133,7 @@ data: } ], "thresholds": "", - "title": "Availability (30d) > 99.000", + "title": "Availability (30d) > 99.000%", "tooltip": { "shared": false }, @@ -138,11 +157,12 @@ data: "dashes": false, "datasource": "$datasource", "decimals": 3, - "fill": 1, + "description": "How much error budget is left looking at our 0.990% availability gurantees?", + "fill": 10, "gridPos": { }, - "id": 3, + "id": 4, "legend": { "alignAsTable": false, "avg": false, @@ -186,7 +206,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "ErrorBudget (30d) > 99.000", + "title": "ErrorBudget (30d) > 99.000%", "tooltip": { "shared": false, "sort": 0, @@ -247,6 +267,7 @@ data: ], "datasource": "$datasource", "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", "format": "percentunit", "gauge": { "maxValue": 100, @@ -258,7 +279,7 @@ data: "gridPos": { }, - "id": 4, + "id": 5, "interval": null, "links": [ @@ -329,11 +350,12 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, "gridPos": { }, - "id": 5, + "id": 6, "legend": { "alignAsTable": false, "avg": false, @@ -357,18 +379,33 @@ data: "renderer": "flot", "repeat": null, "seriesOverrides": [ - + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } ], "spaceLength": 10, "span": 3, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(code_resource:apiserver_request_total:rate5m{verb=\"read\"})", + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "{{`{{`}} code {{`}}`}}", "refId": "A" } ], @@ -420,11 +457,12 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", "fill": 1, "gridPos": { }, - "id": 6, + "id": 7, "legend": { "alignAsTable": false, "avg": false, @@ -490,7 +528,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { @@ -498,7 +536,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true } ] @@ -511,11 +549,12 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", "fill": 1, "gridPos": { }, - "id": 7, + "id": 8, "legend": { "alignAsTable": false, "avg": false, @@ -618,6 +657,7 @@ data: ], "datasource": "$datasource", "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", "format": "percentunit", "gauge": { "maxValue": 100, @@ -629,7 +669,7 @@ data: "gridPos": { }, - "id": 8, + "id": 9, "interval": null, "links": [ @@ -700,11 +740,12 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, "gridPos": { }, - "id": 9, + "id": 10, "legend": { "alignAsTable": false, "avg": false, @@ -713,6 +754,7 @@ data: "min": false, "rightSide": false, "show": true, + "sideWidth": null, "total": false, "values": false }, @@ -728,18 +770,33 @@ data: "renderer": "flot", "repeat": null, "seriesOverrides": [ - + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } ], "spaceLength": 10, "span": 3, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(code_resource:apiserver_request_total:rate5m{verb=\"write\"})", + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "{{`{{`}} code {{`}}`}}", "refId": "A" } ], @@ -791,11 +848,12 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", "fill": 1, "gridPos": { }, - "id": 10, + "id": 11, "legend": { "alignAsTable": false, "avg": false, @@ -861,7 +919,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { @@ -869,7 +927,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true } ] @@ -882,11 +940,12 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", "fill": 1, "gridPos": { }, - "id": 11, + "id": 12, "legend": { "alignAsTable": false, "avg": false, @@ -978,90 +1037,6 @@ data: "collapse": false, "collapsed": false, "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 12, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(up{job=\"apiserver\", cluster=\"$cluster\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Up", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, { "aliasColors": { @@ -1075,222 +1050,6 @@ data: }, "id": 13, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 5, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"2..\", cluster=\"$cluster\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "2xx", - "refId": "A" - }, - { - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"3..\", cluster=\"$cluster\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "3xx", - "refId": "B" - }, - { - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"4..\", cluster=\"$cluster\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "4xx", - "refId": "C" - }, - { - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"5..\", cluster=\"$cluster\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "5xx", - "refId": "D" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "RPC Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 5, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", verb!=\"WATCH\", cluster=\"$cluster\"}[5m])) by (verb, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}verb{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Request duration 99th quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 15, "legend": { "alignAsTable": false, "avg": false, @@ -1317,7 +1076,7 @@ data: ], "spaceLength": 10, - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ @@ -1381,7 +1140,7 @@ data: "gridPos": { }, - "id": 16, + "id": 14, "legend": { "alignAsTable": false, "avg": false, @@ -1408,7 +1167,7 @@ data: ], "spaceLength": 10, - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ @@ -1472,7 +1231,7 @@ data: "gridPos": { }, - "id": 17, + "id": 15, "legend": { "alignAsTable": true, "avg": false, @@ -1499,7 +1258,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 4, "stack": false, "steppedLine": false, "targets": [ @@ -1576,7 +1335,7 @@ data: "gridPos": { }, - "id": 18, + "id": 16, "legend": { "alignAsTable": false, "avg": false, @@ -1667,7 +1426,7 @@ data: "gridPos": { }, - "id": 19, + "id": 17, "legend": { "alignAsTable": false, "avg": false, @@ -1765,7 +1524,7 @@ data: "gridPos": { }, - "id": 20, + "id": 18, "legend": { "alignAsTable": false, "avg": false, @@ -1876,7 +1635,7 @@ data: "gridPos": { }, - "id": 21, + "id": 19, "legend": { "alignAsTable": false, "avg": false, @@ -1967,7 +1726,7 @@ data: "gridPos": { }, - "id": 22, + "id": 20, "legend": { "alignAsTable": false, "avg": false, @@ -2058,7 +1817,7 @@ data: "gridPos": { }, - "id": 23, + "id": 21, "legend": { "alignAsTable": false, "avg": false, diff --git a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml index f9f9627dc5..df3bf8ae3c 100644 --- a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml +++ b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -48,6 +48,7 @@ data: "fill": 1, "format": "percentunit", "id": 1, + "interval": "1m", "legend": { "avg": false, "current": false, @@ -856,7 +857,7 @@ data: ], "targets": [ { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_owner{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1275,7 +1276,7 @@ data: ], "targets": [ { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_owner{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1401,6 +1402,7 @@ data: "datasource": "$datasource", "fill": 1, "id": 11, + "interval": "1m", "legend": { "avg": false, "current": false, diff --git a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml index 2823e136a3..5855357704 100644 --- a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml +++ b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -1276,6 +1276,7 @@ data: "datasource": "$datasource", "fill": 1, "id": 9, + "interval": "1m", "legend": { "avg": false, "current": false, diff --git a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-node.yaml index 41d131f424..b2c3752ed0 100644 --- a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-node.yaml +++ b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -75,7 +75,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -304,7 +304,7 @@ data: ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -313,7 +313,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -322,7 +322,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -331,7 +331,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -340,7 +340,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -440,7 +440,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\", container!=\"\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -723,7 +723,7 @@ data: ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -732,7 +732,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -741,7 +741,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{node=\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -750,7 +750,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -759,7 +759,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{node=\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -768,7 +768,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -777,7 +777,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -786,7 +786,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -905,7 +905,7 @@ data: "hide": 0, "includeAll": false, "label": null, - "multi": false, + "multi": true, "name": "node", "options": [ diff --git a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml index b3a557d5f0..4716f79791 100644 --- a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml +++ b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -1031,6 +1031,7 @@ data: "datasource": "$datasource", "fill": 10, "id": 6, + "interval": "1m", "legend": { "avg": false, "current": false, @@ -1129,6 +1130,7 @@ data: "datasource": "$datasource", "fill": 10, "id": 7, + "interval": "1m", "legend": { "avg": false, "current": false, @@ -1227,6 +1229,7 @@ data: "datasource": "$datasource", "fill": 10, "id": 8, + "interval": "1m", "legend": { "avg": false, "current": false, @@ -1325,6 +1328,7 @@ data: "datasource": "$datasource", "fill": 10, "id": 9, + "interval": "1m", "legend": { "avg": false, "current": false, @@ -1423,6 +1427,7 @@ data: "datasource": "$datasource", "fill": 10, "id": 10, + "interval": "1m", "legend": { "avg": false, "current": false, @@ -1521,6 +1526,7 @@ data: "datasource": "$datasource", "fill": 10, "id": 11, + "interval": "1m", "legend": { "avg": false, "current": false, diff --git a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml index 4e3cbe9e99..1fd37c1a4d 100644 --- a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml +++ b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -777,6 +777,7 @@ data: "datasource": "$datasource", "fill": 1, "id": 5, + "interval": "1m", "legend": { "avg": false, "current": false, diff --git a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml index 608ebf068e..96faac364f 100644 --- a/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml +++ b/stable/prometheus-operator/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -937,6 +937,7 @@ data: "datasource": "$datasource", "fill": 1, "id": 5, + "interval": "1m", "legend": { "avg": false, "current": false, @@ -1963,7 +1964,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod) \ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}workload{{`}}`}}", diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/alertmanager.rules.yaml index 1c6db4095d..bed2c04aff 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/alertmanager.rules.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -29,8 +29,16 @@ spec: rules: - alert: AlertmanagerConfigInconsistent annotations: - message: The configuration of the instances of the Alertmanager cluster `{{`{{`}}$labels.service{{`}}`}}` are out of sync. - expr: count_values("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) BY (service) / ON(service) GROUP_LEFT() label_replace(max(prometheus_operator_spec_replicas{job="{{ $operatorJob }}",namespace="{{ $namespace }}",controller="alertmanager"}) by (name, job, namespace, controller), "service", "$1", "name", "(.*)") != 1 + message: 'The configuration of the instances of the Alertmanager cluster `{{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.service {{`}}`}}` are out of sync. + + {{`{{`}} range printf "alertmanager_config_hash{namespace=\"%s\",service=\"%s\"}" $labels.namespace $labels.service | query {{`}}`}} + + Configuration hash for pod {{`{{`}} .Labels.pod {{`}}`}} is "{{`{{`}} printf "%.f" .Value {{`}}`}}" + + {{`{{`}} end {{`}}`}} + + ' + expr: count by(namespace,service) (count_values by(namespace,service) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"})) != 1 for: 5m labels: severity: critical diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/k8s.rules.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/k8s.rules.yaml index 4bc2cc7df5..720ffabe6e 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/k8s.rules.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -30,31 +30,31 @@ spec: sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( - 1, max by(cluster, namespace, pod, node) (kube_pod_info) + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate - expr: |- container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, - max by(namespace, pod, node) (kube_pod_info) + max by(namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_memory_working_set_bytes - expr: |- container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, - max by(namespace, pod, node) (kube_pod_info) + max by(namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_memory_rss - expr: |- container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, - max by(namespace, pod, node) (kube_pod_info) + max by(namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_memory_cache - expr: |- container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, - max by(namespace, pod, node) (kube_pod_info) + max by(namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_memory_swap - expr: sum(container_memory_usage_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}) by (namespace) diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100644 index 0000000000..25b2bf08de --- /dev/null +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,146 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/coreos/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/helm/charts/tree/master/stable/prometheus-operator/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "prometheus-operator.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "prometheus-operator.namespace" . }} + labels: + app: {{ template "prometheus-operator.name" . }} +{{ include "prometheus-operator.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: |- + 1 - ( + ( + # write too slow + sum(increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d])) + - + sum(increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d])) + ) + + ( + # read too slow + sum(increase(apiserver_request_duration_seconds_count{verb=~"LIST|GET"}[30d])) + - + ( + sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30d])) + + sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="namespace",le="0.5"}[30d])) + + sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="cluster",le="5"}[30d])) + ) + ) + + # errors + sum(code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum(code:apiserver_request_total:increase30d) + labels: + verb: all + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum(increase(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30d])) + - + ( + # too slow + sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30d])) + + sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30d])) + + sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30d])) + ) + + + # errors + sum(code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum(code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum(increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d])) + - + sum(increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d])) + ) + + + # errors + sum(code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum(code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + record: apiserver_request:availability30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + record: code:apiserver_request_total:increase30d + - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + record: code:apiserver_request_total:increase30d +{{- end }} \ No newline at end of file diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml index 010d344616..5549bfb191 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -34,7 +34,9 @@ spec: sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) for: 2m labels: + long: 1h severity: critical + short: 5m - alert: KubeAPIErrorBudgetBurn annotations: message: The API server is burning too much error budget @@ -45,7 +47,9 @@ spec: sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) for: 15m labels: + long: 6h severity: critical + short: 30m - alert: KubeAPIErrorBudgetBurn annotations: message: The API server is burning too much error budget @@ -56,7 +60,9 @@ spec: sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) for: 1h labels: + long: 1d severity: warning + short: 2h - alert: KubeAPIErrorBudgetBurn annotations: message: The API server is burning too much error budget @@ -67,5 +73,7 @@ spec: sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) for: 3h labels: + long: 3d severity: warning + short: 6h {{- end }} \ No newline at end of file diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml index 1b00134e87..1e0645465e 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml @@ -31,7 +31,7 @@ spec: sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[1d])) - ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="resource",le="0.1"}[1d])) + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[1d])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[1d])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[1d])) ) @@ -52,7 +52,7 @@ spec: sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[1h])) - ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="resource",le="0.1"}[1h])) + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[1h])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[1h])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[1h])) ) @@ -73,7 +73,7 @@ spec: sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[2h])) - ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="resource",le="0.1"}[2h])) + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[2h])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[2h])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[2h])) ) @@ -94,7 +94,7 @@ spec: sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30m])) - ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="resource",le="0.1"}[30m])) + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30m])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30m])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30m])) ) @@ -115,7 +115,7 @@ spec: sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[3d])) - ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="resource",le="0.1"}[3d])) + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[3d])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[3d])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[3d])) ) @@ -136,7 +136,7 @@ spec: sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[5m])) - ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="resource",le="0.1"}[5m])) + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[5m])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[5m])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[5m])) ) @@ -157,7 +157,7 @@ spec: sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[6h])) - ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="resource",le="0.1"}[6h])) + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[6h])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[6h])) + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[6h])) ) @@ -283,78 +283,6 @@ spec: labels: verb: write record: apiserver_request:burnrate6h - - expr: |- - 1 - ( - ( - # write too slow - sum(increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d])) - - - sum(increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d])) - ) + - ( - # read too slow - sum(increase(apiserver_request_duration_seconds_count{verb=~"LIST|GET"}[30d])) - - - ( - sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="resource",le="0.1"}[30d])) + - sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="namespace",le="0.5"}[30d])) + - sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="cluster",le="5"}[30d])) - ) - ) + - # errors - sum(code:apiserver_request_total:increase30d{code=~"5.."}) - ) - / - sum(code:apiserver_request_total:increase30d) - labels: - verb: all - record: apiserver_request:availability30d - - expr: |- - 1 - ( - sum(increase(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30d])) - - - ( - # too slow - sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="resource",le="0.1"}[30d])) + - sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30d])) + - sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30d])) - ) - + - # errors - sum(code:apiserver_request_total:increase30d{verb="read",code=~"5.."}) - ) - / - sum(code:apiserver_request_total:increase30d{verb="read"}) - labels: - verb: read - record: apiserver_request:availability30d - - expr: |- - 1 - ( - ( - # too slow - sum(increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d])) - - - sum(increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d])) - ) - + - # errors - sum(code:apiserver_request_total:increase30d{verb="write",code=~"5.."}) - ) - / - sum(code:apiserver_request_total:increase30d{verb="write"}) - labels: - verb: write - record: apiserver_request:availability30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver"}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) - labels: - verb: read - record: code:apiserver_request_total:increase30d - - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) - labels: - verb: write - record: code:apiserver_request_total:increase30d - expr: sum by (code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) labels: verb: read diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-apps.yaml index 3ae09119ae..1b9206126d 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-apps.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -29,18 +29,25 @@ spec: annotations: message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is restarting {{`{{`}} printf "%.2f" $value {{`}}`}} times / 5 minutes. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping - expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[15m]) * 60 * 5 > 0 + expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) * 60 * 5 > 0 for: 15m labels: - severity: critical + severity: warning - alert: KubePodNotReady annotations: message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodnotready - expr: sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown"}) * on(namespace, pod) group_left(owner_kind) max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})) > 0 + expr: |- + sum by (namespace, pod) ( + max by(namespace, pod) ( + kube_pod_status_phase{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown"} + ) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) ( + 1, max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 for: 15m labels: - severity: critical + severity: warning - alert: KubeDeploymentGenerationMismatch annotations: message: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. @@ -51,7 +58,7 @@ spec: kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} for: 15m labels: - severity: critical + severity: warning - alert: KubeDeploymentReplicasMismatch annotations: message: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. @@ -68,7 +75,7 @@ spec: ) for: 15m labels: - severity: critical + severity: warning - alert: KubeStatefulSetReplicasMismatch annotations: message: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. @@ -85,7 +92,7 @@ spec: ) for: 15m labels: - severity: critical + severity: warning - alert: KubeStatefulSetGenerationMismatch annotations: message: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. @@ -96,26 +103,32 @@ spec: kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} for: 15m labels: - severity: critical + severity: warning - alert: KubeStatefulSetUpdateNotRolledOut annotations: message: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetupdatenotrolledout expr: |- - max without (revision) ( - kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - unless - kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - ) - * ( - kube_statefulset_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - != - kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + max without (revision) ( + kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 ) for: 15m labels: - severity: critical + severity: warning - alert: KubeDaemonSetRolloutStuck annotations: message: Only {{`{{`}} $value | humanizePercentage {{`}}`}} of the desired Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are scheduled and ready. @@ -126,7 +139,7 @@ spec: kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} < 1.00 for: 15m labels: - severity: critical + severity: warning - alert: KubeContainerWaiting annotations: message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. @@ -154,14 +167,6 @@ spec: for: 15m labels: severity: warning - - alert: KubeCronJobRunning - annotations: - message: CronJob {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.cronjob {{`}}`}} is taking more than 1h to complete. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecronjobrunning - expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 3600 - for: 1h - labels: - severity: warning - alert: KubeJobCompletion annotations: message: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than one hour to complete. diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-resources.yaml index 0247f5ebc5..dc80dc62cc 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-resources.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -76,18 +76,18 @@ spec: for: 5m labels: severity: warning - - alert: KubeQuotaExceeded + - alert: KubeQuotaFullyUsed annotations: message: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubequotaexceeded + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubequotafullyused expr: |- kube_resourcequota{job="kube-state-metrics", type="used"} / ignoring(instance, job, type) (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) - > 0.90 + >= 1 for: 15m labels: - severity: warning + severity: info - alert: CPUThrottlingHigh annotations: message: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml index 7583a599b8..7cbb75f3c5 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -24,40 +24,6 @@ spec: groups: - name: kubernetes-system-apiserver rules: - - alert: KubeAPILatencyHigh - annotations: - message: The API server has an abnormal latency of {{`{{`}} $value {{`}}`}} seconds for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}}. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh - expr: |- - ( - cluster:apiserver_request_duration_seconds:mean5m{job="apiserver"} - > - on (verb) group_left() - ( - avg by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job="apiserver"} >= 0) - + - 2*stddev by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job="apiserver"} >= 0) - ) - ) > on (verb) group_left() - 1.2 * avg by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job="apiserver"} >= 0) - and on (verb,resource) - cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job="apiserver",quantile="0.99"} - > - 1 - for: 5m - labels: - severity: warning - - alert: KubeAPIErrorsHigh - annotations: - message: API server is returning errors for {{`{{`}} $value | humanizePercentage {{`}}`}} of requests for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}} {{`{{`}} $labels.subresource {{`}}`}}. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh - expr: |- - sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) by (resource,subresource,verb) - / - sum(rate(apiserver_request_total{job="apiserver"}[5m])) by (resource,subresource,verb) > 0.05 - for: 10m - labels: - severity: warning - alert: KubeClientCertificateExpiration annotations: message: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days. diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/node-exporter.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/node-exporter.yaml index 7b6c601ba3..9e10a57712 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/node-exporter.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/node-exporter.yaml @@ -164,12 +164,20 @@ spec: severity: warning - alert: NodeHighNumberConntrackEntriesUsed annotations: - description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used' + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodehighnumberconntrackentriesused - summary: Number of conntrack are getting close to the limit + summary: Number of conntrack are getting close to the limit. expr: (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75 labels: severity: warning + - alert: NodeTextFileCollectorScrapeError + annotations: + description: Node Exporter text file collector failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: warning - alert: NodeClockSkewDetected annotations: message: Clock on {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 300s. Ensure NTP is configured correctly on this host. diff --git a/stable/prometheus-operator/templates/prometheus/rules-1.14/node.rules.yaml b/stable/prometheus-operator/templates/prometheus/rules-1.14/node.rules.yaml index 75ab27edf6..1e754b2001 100644 --- a/stable/prometheus-operator/templates/prometheus/rules-1.14/node.rules.yaml +++ b/stable/prometheus-operator/templates/prometheus/rules-1.14/node.rules.yaml @@ -24,12 +24,12 @@ spec: groups: - name: node.rules rules: - - expr: sum(min(kube_pod_info) by (cluster, node)) + - expr: sum(min(kube_pod_info{node!=""}) by (cluster, node)) record: ':kube_pod_info_node_count:' - expr: |- topk by(namespace, pod) (1, max by (node, namespace, pod) ( - label_replace(kube_pod_info{job="kube-state-metrics"}, "pod", "$1", "pod", "(.*)") + label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)") )) record: 'node_namespace_pod:kube_pod_info:' - expr: |-