Exclude high cardinality cAdvisor metrics

charts/flagger/templates/prometheus.yaml

@@ -133,38 +133,22 @@ data:
       scheme: https
       tls_config:
         ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        insecure_skip_verify: true
       bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
       relabel_configs:
       - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
         action: keep
         regex: kubernetes;https
 
-    # Scrape config for nodes
-    - job_name: 'kubernetes-nodes'
-      scheme: https
-      tls_config:
-        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-      kubernetes_sd_configs:
-      - role: node
-      relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      - target_label: __address__
-        replacement: kubernetes.default.svc:443
-      - source_labels: [__meta_kubernetes_node_name]
-        regex: (.+)
-        target_label: __metrics_path__
-        replacement: /api/v1/nodes/${1}/proxy/metrics
-
     # scrape config for cAdvisor
     - job_name: 'kubernetes-cadvisor'
       scheme: https
       tls_config:
         ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        insecure_skip_verify: true
       bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
       kubernetes_sd_configs:
-      - role: node
+        - role: node
       relabel_configs:
       - action: labelmap
         regex: __meta_kubernetes_node_label_(.+)
@@ -174,6 +158,14 @@ data:
         regex: (.+)
         target_label: __metrics_path__
         replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+      # exclude high cardinality metrics
+      metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: (container|machine)_(cpu|memory|network|fs)_(.+)
+        action: keep
+      - source_labels: [__name__]
+        regex: container_memory_failures_total
+        action: drop
 
     # scrape config for pods
     - job_name: kubernetes-pods

kustomize/base/prometheus/prometheus.yml

@@ -2,12 +2,11 @@ global:
   scrape_interval: 5s
 scrape_configs:
 
-# Scrape config for AppMesh Envoy sidecar
+# scrape config for AppMesh Envoy sidecar
 - job_name: 'appmesh-envoy'
   metrics_path: /stats/prometheus
   kubernetes_sd_configs:
   - role: pod
-
   relabel_configs:
   - source_labels: [__meta_kubernetes_pod_container_name]
     action: keep
@@ -25,8 +24,7 @@ scrape_configs:
   - source_labels: [__meta_kubernetes_pod_name]
     action: replace
     target_label: kubernetes_pod_name
-
-  # Exclude high cardinality metrics
+  # exclude high cardinality metrics
   metric_relabel_configs:
   - source_labels: [ cluster_name ]
     regex: '(outbound|inbound|prometheus_stats).*'
@@ -56,7 +54,7 @@ scrape_configs:
     regex: 'envoy_cluster_(lb|retry|bind|internal|max|original).*'
     action: drop
 
-# Scrape config for API servers
+# scrape config for API servers
 - job_name: 'kubernetes-apiservers'
   kubernetes_sd_configs:
   - role: endpoints
@@ -66,35 +64,19 @@ scrape_configs:
   scheme: https
   tls_config:
     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecure_skip_verify: true
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   relabel_configs:
   - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
     action: keep
     regex: kubernetes;https
 
-# Scrape config for nodes
-- job_name: 'kubernetes-nodes'
-  scheme: https
-  tls_config:
-    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-  kubernetes_sd_configs:
-  - role: node
-  relabel_configs:
-  - action: labelmap
-    regex: __meta_kubernetes_node_label_(.+)
-  - target_label: __address__
-    replacement: kubernetes.default.svc:443
-  - source_labels: [__meta_kubernetes_node_name]
-    regex: (.+)
-    target_label: __metrics_path__
-    replacement: /api/v1/nodes/${1}/proxy/metrics
-
 # scrape config for cAdvisor
 - job_name: 'kubernetes-cadvisor'
   scheme: https
   tls_config:
     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecure_skip_verify: true
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   kubernetes_sd_configs:
   - role: node
@@ -107,6 +89,14 @@ scrape_configs:
     regex: (.+)
     target_label: __metrics_path__
     replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+  # exclude high cardinality metrics
+  metric_relabel_configs:
+  - source_labels: [__name__]
+    regex: (container|machine)_(cpu|memory|network|fs)_(.+)
+    action: keep
+  - source_labels: [__name__]
+    regex: container_memory_failures_total
+    action: drop
 
 # scrape config for pods
 - job_name: kubernetes-pods