krkn/config/metrics-aggregated.yaml

metrics:
# API server
  - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0
    metricName: APIRequestRate
    instant: True

  - query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0
    metricName: APIInflightRequests
    instant: True

  - query: histogram_quantile(0.99, rate(apiserver_current_inflight_requests[5m]))
    metricName: APIInflightRequests
    instant: True

# Container & pod metrics
  - query: (sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0
    metricName: containerMemory-Masters
    instant: true

  - query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|.*apiserver|authentication|.*controller-manager|.*scheduler)"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0
    metricName: containerCPU-Masters
    instant: true

  - query: (sum(irate(container_cpu_usage_seconds_total{pod!="",container="prometheus",namespace="openshift-monitoring"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0
    metricName: containerCPU-Prometheus
    instant: true

  - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) by (namespace, container)) > 0
    metricName: containerCPU-AggregatedWorkers
    instant: true

  - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"}[2m]) * 100 and on (node) kube_node_role{role="infra"}) by (namespace, container)) > 0
    metricName: containerCPU-AggregatedInfra

  - query: (sum(container_memory_rss{pod!="",namespace="openshift-monitoring",name!="",container="prometheus"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0
    metricName: containerMemory-Prometheus
    instant: True

  - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"} and on (node) kube_node_role{role="worker"}) by (container, namespace)
    metricName: containerMemory-AggregatedWorkers
    instant: True

  - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"} and on (node) kube_node_role{role="infra"}) by (container, namespace)
    metricName: containerMemory-AggregatedInfra
    instant: True

# Node metrics
  - query: (sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")) > 0
    metricName: nodeCPU-Masters
    instant: True

  - query: max(max_over_time(sum(irate(node_cpu_seconds_total{mode!="idle", mode!="steal"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")) by (instance)[.elapsed:]))
    metricName: maxCPU-Masters
    instant: true
  
  - query: avg(avg_over_time((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes)[.elapsed:]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)"))
    metricName: nodeMemory-Masters
    instant: true

  - query: (avg((sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)"))) by (mode)) > 0
    metricName: nodeCPU-AggregatedWorkers
    instant: True

  - query: (avg((sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)"))) by (mode)) > 0
    metricName: nodeCPU-AggregatedInfra
    instant: True

  - query: avg(avg_over_time((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes)[.elapsed:]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)"))
    metricName: nodeMemory-Masters
    instant: true

  - query: max(max_over_time((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes)[.elapsed:]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)"))
    metricName: maxMemory-Masters
    instant: true

  - query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)"))
    metricName: nodeMemoryAvailable-AggregatedWorkers
    instant: True

  - query: max(max_over_time(sum(irate(node_cpu_seconds_total{mode!="idle", mode!="steal"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (instance)[.elapsed:]))
    metricName: maxCPU-Workers
    instant: true
  
  - query: max(max_over_time((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes)[.elapsed:]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)"))
    metricName: maxMemory-Workers
    instant: true

  - query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)"))
    metricName: nodeMemoryAvailable-AggregatedInfra
    instant: True

  - query: avg(node_memory_Active_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")
    metricName: nodeMemoryActive-Masters
    instant: True

  - query: avg(node_memory_Active_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)"))
    metricName: nodeMemoryActive-AggregatedWorkers
    instant: True

  - query: avg(avg(node_memory_Active_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)"))
    metricName: nodeMemoryActive-AggregatedInfra
    instant: True

# Etcd metrics
  - query: sum(rate(etcd_server_leader_changes_seen_total[2m]))
    metricName: etcdLeaderChangesRate
    instant: True

  - query: etcd_server_is_leader > 0
    metricName: etcdServerIsLeader
    instant: True

  - query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m]))
    metricName: 99thEtcdDiskBackendCommitDurationSeconds
    instant: True

  - query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m]))
    metricName: 99thEtcdDiskWalFsyncDurationSeconds
    instant: True

  - query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m]))
    metricName: 99thEtcdRoundTripTimeSeconds
    instant: True

  - query: sum by (cluster_version)(etcd_cluster_version)
    metricName: etcdVersion
    instant: true

  - query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0
    metricName: etcdObjectCount
    instant: True

  - query: histogram_quantile(0.99,sum(rate(etcd_request_duration_seconds_bucket[2m])) by (le,operation,apiserver)) > 0
    metricName: P99APIEtcdRequestLatency
    instant: True

  - query: sum by (instance) (apiserver_storage_objects)
    metricName: etcdTotalObjectCount 
    instant: True

  - query: topk(500, max by(resource) (apiserver_storage_objects))
    metricName: etcdTopObectCount    
    instant: True