From dd4445f178cb738c7d93d671d28d38eda5847576 Mon Sep 17 00:00:00 2001 From: Cristian Klein Date: Thu, 16 Jan 2020 10:02:37 +0100 Subject: [PATCH] [stable/prometheus] prometheus.io/scrape-slow for scraping slow target (#19930) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * prometheus.io/scrape-slow for scraping slow target Background ========== Some users require scraping slow targets -- such as [elasticsearch-exporter](https://github.com/helm/charts/tree/master/stable/elasticsearch-exporter)) -- with a larger timeout and larger interval. Globally changing the scrape interval and timeout is unsatisfactory, since this would reduce sampling rates for targets that are fast. A [Prometheus feature](https://github.com/prometheus/prometheus/issues/2353) was proposed to set the scraping timeout and interval via annotations. However, this would require considerable re-engineering, since scrape timeout and interval is a per-job configuration and not a per-target one. Approach ======== This PR proposed the `prometheus.io/scrape-slow` annotation for scraping endpoints that are considerably slower, i.e., with an interval of 5 minutes and a timeout of 30 seconds. Other annotations are identical to the `prometheus.io/scrape` annotation what users are already familiar with. Co-authored-by: Emil Vannebäck Signed-off-by: Cristian Klein * Bump stable/prometheus version Signed-off-by: Cristian Klein Co-authored-by: Emil --- stable/prometheus/Chart.yaml | 2 +- stable/prometheus/values.yaml | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/stable/prometheus/Chart.yaml b/stable/prometheus/Chart.yaml index ff44e13149..3cf1b3ebba 100644 --- a/stable/prometheus/Chart.yaml +++ b/stable/prometheus/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: prometheus -version: 10.2.0 +version: 10.3.0 appVersion: 2.15.2 description: Prometheus is a monitoring system and time series database. home: https://prometheus.io/ diff --git a/stable/prometheus/values.yaml b/stable/prometheus/values.yaml index 3ed0fd5b19..d92f23125d 100644 --- a/stable/prometheus/values.yaml +++ b/stable/prometheus/values.yaml @@ -1369,6 +1369,55 @@ serverFiles: action: replace target_label: kubernetes_node + # Scrape config for slow service endpoints; same as above, but with a larger + # timeout and a larger interval + # + # The relabeling allows the actual service scrape endpoint to be configured + # via the following annotations: + # + # * `prometheus.io/scrape-slow`: Only scrape services that have a value of `true` + # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need + # to set this to `https` & most likely set the `tls_config` of the scrape config. + # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. + # * `prometheus.io/port`: If the metrics are exposed on a different port to the + # service then set this appropriately. + - job_name: 'kubernetes-service-endpoints-slow' + + scrape_interval: 5m + scrape_timeout: 30s + + kubernetes_sd_configs: + - role: endpoints + + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape_slow] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name + - source_labels: [__meta_kubernetes_pod_node_name] + action: replace + target_label: kubernetes_node + - job_name: 'prometheus-pushgateway' honor_labels: true