github/node-problem-detector
1
0
Fork 0
mirror of https://github.com/kubernetes/node-problem-detector.git synced 2026-08-19 04:06:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Collect more cpu/disk/memory metrics

Browse Source
This commit is contained in:
Xuewei Zhang
2020-02-03 15:29:45 -08:00
parent 9ade82734d
commit 83b09277f0
10 changed files with 655 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/system-stats-monitor.json
+44
View File
@@ -1,4 +1,14 @@
{
"cpu": {
"metricsConfigs": {
"cpu/runnable_task_count": {
"displayName": "cpu/runnable_task_count"
},
"cpu/usage_time": {
"displayName": "cpu/usage_time"
}
}
},
"disk": {
"metricsConfigs": {
"disk/io_time": {
@@ -9,6 +19,21 @@
},
"disk/avg_queue_len": {
"displayName": "disk/avg_queue_len"
},
"disk/operation_count": {
"displayName": "disk/operation_count"
},
"disk/merged_operation_count": {
"displayName": "disk/merged_operation_count"
},
"disk/operation_bytes_count": {
"displayName": "disk/operation_bytes_count"
},
"disk/operation_time": {
"displayName": "disk/operation_time"
},
"disk/bytes_used": {
"displayName": "disk/bytes_used"
}
},
"includeRootBlk": true,
@@ -22,5 +47,24 @@
}
}
},
"memory": {
"metricsConfigs": {
"memory/bytes_used": {
"displayName": "memory/bytes_used"
},
"memory/anonymous_used": {
"displayName": "memory/anonymous_used"
},
"memory/page_cache_used": {
"displayName": "memory/page_cache_used"
},
"memory/unevictable_used": {
"displayName": "memory/unevictable_used"
},
"memory/dirty_used": {
"displayName": "memory/dirty_used"
}
}
},
"invokeInterval": "60s"
}
pkg/exporters/stackdriver/stackdriver_exporter.go
+18 -6
View File
@@ -47,12 +47,24 @@ func init() {
const exporterName = "stackdriver"
var NPDMetricToSDMetric = map[metrics.MetricID]string{
metrics.HostUptimeID: "compute.googleapis.com/guest/system/uptime",
metrics.ProblemCounterID: "compute.googleapis.com/guest/system/problem_count",
metrics.ProblemGaugeID: "compute.googleapis.com/guest/system/problem_state",
metrics.DiskAvgQueueLenID: "compute.googleapis.com/guest/disk/queue_length",
metrics.DiskIOTimeID: "compute.googleapis.com/guest/disk/io_time",
metrics.DiskWeightedIOID: "compute.googleapis.com/guest/disk/weighted_io_time",
metrics.CPURunnableTaskCountID: "compute.googleapis.com/guest/cpu/runnable_task_count",
metrics.CPUUsageTimeID: "compute.googleapis.com/guest/cpu/usage_time",
metrics.DiskAvgQueueLenID: "compute.googleapis.com/guest/disk/queue_length",
metrics.DiskBytesUsedID: "compute.googleapis.com/guest/disk/bytes_used",
metrics.DiskIOTimeID: "compute.googleapis.com/guest/disk/io_time",
metrics.DiskMergedOpsCountID: "compute.googleapis.com/guest/disk/merged_operation_count",
metrics.DiskOpsBytesID: "compute.googleapis.com/guest/disk/operation_bytes_count",
metrics.DiskOpsCountID: "compute.googleapis.com/guest/disk/operation_count",
metrics.DiskOpsTimeID: "compute.googleapis.com/guest/disk/operation_time",
metrics.DiskWeightedIOID: "compute.googleapis.com/guest/disk/weighted_io_time",
metrics.HostUptimeID: "compute.googleapis.com/guest/system/uptime",
metrics.MemoryAnonymousUsedID: "compute.googleapis.com/guest/memory/anonymous_used",
metrics.MemoryBytesUsedID: "compute.googleapis.com/guest/memory/bytes_used",
metrics.MemoryDirtyUsedID: "compute.googleapis.com/guest/memory/dirty_used",
metrics.MemoryPageCacheUsedID: "compute.googleapis.com/guest/memory/page_cache_used",
metrics.MemoryUnevictableUsedID: "compute.googleapis.com/guest/memory/unevictable_used",
metrics.ProblemCounterID: "compute.googleapis.com/guest/system/problem_count",
metrics.ProblemGaugeID: "compute.googleapis.com/guest/system/problem_state",
}
func getMetricTypeConversionFunction(customMetricPrefix string) func(*view.View) string {
pkg/systemstatsmonitor/README.md
+46 -6
View File
@@ -4,27 +4,67 @@
Currently supported components are:
* cpu
* disk
* host
* memory
See example config file [here](https://github.com/kubernetes/node-problem-detector/blob/master/config/system-stats-monitor.json).
By setting the `metricsConfigs` field and `displayName` field ([example](https://github.com/kubernetes/node-problem-detector/blob/master/config/system-stats-monitor.json)), you can specify the list of metrics to be collected, and their display names on the Prometheus scaping endpoint.
## Detailed Configuration Options
### Global Configurations
Data collection period can be specified globally in the config file, see `invokeInterval` at the [example](https://github.com/kubernetes/node-problem-detector/blob/master/config/system-stats-monitor.json).
### CPU
Below metrics are collected from `cpu` component:
* `cpu_runnable_task_count`: The average number of runnable tasks in the run-queue during the last minute. Collected from [`/proc/loadavg`][/proc doc].
* `cpu_usage_time`: CPU usage, in seconds. The [CPU state][/proc doc] for the corresponding usage is reported under the `state` metric label (e.g. `user`, `nice`, `system`...).
[/proc doc]: http://man7.org/linux/man-pages/man5/proc.5.html
### Disk
Below metrics are collected from `disk` component:
* `disk/io_time`: [# of milliseconds spent doing I/Os on this device](https://www.kernel.org/doc/Documentation/iostats.txt)
* `disk/weighted_io`: [# of milliseconds spent doing I/Os on this device](https://www.kernel.org/doc/Documentation/iostats.txt)
* `disk/avg_queue_len`: [average # of requests that was waiting in queue or being serviced during the last `invokeInterval`](https://www.xaprb.com/blog/2010/01/09/how-linux-iostat-computes-its-results/)
* `disk_io_time`: [# of milliseconds spent doing I/Os on this device][iostat doc]
* `disk_weighted_io`: [# of milliseconds spent doing I/Os on this device][iostat doc]
* `disk_avg_queue_len`: [average # of requests that was waiting in queue or being serviced during the last `invokeInterval`](https://www.xaprb.com/blog/2010/01/09/how-linux-iostat-computes-its-results/)
* `disk_operation_count`: [# of reads/writes completed][iostat doc]
* `disk_merged_operation_count`: [# of reads/writes merged][iostat doc]
* `disk_operation_bytes_count`: # of Bytes used for reads/writes on this device
* `disk_operation_time`: [# of milliseconds spent reading/writing][iostat doc]
* `disk_bytes_used`: Disk usage in Bytes. The usage state is reported under the `state` metric label (e.g. `used`, `free`). Summing values of all states yields the disk size.
By setting the `metricsConfigs` field and `displayName` field ([example](https://github.com/kubernetes/node-problem-detector/blob/master/config/system-stats-monitor.json)), you can specify the list of metrics to be collected, and their display names on the Prometheus scaping endpoint. The name of the disk block device will be reported in the `device` metrics label.
The name of the disk block device is reported in the `device_name` metric label (e.g. `sda`).
For the metrics that separates read/write operations, the IO direction is reported in the `direction` metric label (e.g. `read`, `write`).
And a few other options:
* `includeRootBlk`: When set to `true`, add all block devices that's [not a slave or holder device](http://man7.org/linux/man-pages/man8/lsblk.8.html) to the list of disks that System Stats Monitor collects metrics from. When set to `false`, do not modify the list of disks that System Stats Monitor collects metrics from.
* `includeRootBlk`: When set to `true`, add all block devices that's [not a slave or holder device][lsblk doc] to the list of disks that System Stats Monitor collects metrics from. When set to `false`, do not modify the list of disks that System Stats Monitor collects metrics from.
* `includeAllAttachedBlk`: When set to `true`, add all currently attached block devices to the list of disks that System Stats Monitor collects metrics from. When set to `false`, do not modify the list of disks that System Stats Monitor collects metrics from.
* `lsblkTimeout`: System Stats Monitor uses [`lsblk`](http://man7.org/linux/man-pages/man8/lsblk.8.html) to retrieve block devices information. This option sets the timeout for calling `lsblk` commands.
* `lsblkTimeout`: System Stats Monitor uses [`lsblk`][lsblk doc] to retrieve block devices information. This option sets the timeout for calling `lsblk` commands.
[iostat doc]: https://www.kernel.org/doc/Documentation/iostats.txt
[lsblk doc]: http://man7.org/linux/man-pages/man8/lsblk.8.html
### Host
Below metrics are collected from `host` component:
* `host_uptime`: The uptime of the operating system, in seconds. OS version and kernel versions are reported under the `os_version` and `kernel_version` metric label (e.g. `cos 73-11647.217.0`, `4.14.127+`).
### Memory
Below metrics are collected from `memory` component:
* `memory_bytes_used`: Memory usage by each memory state, in Bytes. The memory state is reported under the `state` metric label (e.g. `free`, `used`, `buffered`...). Summing values of all states yields the total memory of the node.
* `memory_anonymous_used`: Anonymous memory usage, in Bytes. Memory usage state is reported under the `state` metric label (e.g. `active`, `inactive`). `active` means the memory has been used more recently and usually not swapped until needed. Summing values of all states yields the total anonymous memory used.
* `memory_page_cache_used`: Page cache memory usage, in Bytes. Memory usage state is reported under the `state` metric label (e.g. `active`, `inactive`). `active` means the memory has been used more recently and usually not reclaimed until needed. Summing values of all states yields the total page cache memory used.
* `memory_unevictable_used`: [Unevictable memory][/proc doc] usage, in Bytes.
* `memory_dirty_used`: Dirty pages usage, in Bytes. Memory usage state is reported under the `state` metric label (e.g. `dirty`, `writeback`). `dirty` means the memory is waiting to be written back to disk, and `writeback` means the memory is actively being written back to disk.
pkg/systemstatsmonitor/cpu_collector.go
+144
View File
@@ -0,0 +1,144 @@
/*
Copyright 2020 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package systemstatsmonitor
import (
"github.com/golang/glog"
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/load"
ssmtypes "k8s.io/node-problem-detector/pkg/systemstatsmonitor/types"
"k8s.io/node-problem-detector/pkg/util/metrics"
)
// clockTick is the ratio between 1 second and 1 USER_HZ (a clock tick).
//
// CLK_TCK is 100 in most architectures. If NPD ever runs on a super special architecture,
// we can work out a way to detect the clock tick on that architecture (might require
// cross-compilation with C library or parsing kernel ABIs). For now, it's not worth the
// complexity.
//
// See documentation at http://man7.org/linux/man-pages/man5/proc.5.html
const clockTick float64 = 100.0
type cpuCollector struct {
mRunnableTaskCount *metrics.Float64Metric
mUsageTime *metrics.Float64Metric
config *ssmtypes.CPUStatsConfig
lastUsageTime map[string]float64
}
func NewCPUCollectorOrDie(cpuConfig *ssmtypes.CPUStatsConfig) *cpuCollector {
cc := cpuCollector{config: cpuConfig}
var err error
cc.mRunnableTaskCount, err = metrics.NewFloat64Metric(
metrics.CPURunnableTaskCountID,
cpuConfig.MetricsConfigs[string(metrics.CPURunnableTaskCountID)].DisplayName,
"The average number of runnable tasks in the run-queue during the last minute",
"1",
metrics.LastValue,
[]string{})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.CPURunnableTaskCountID, err)
}
cc.mUsageTime, err = metrics.NewFloat64Metric(
metrics.CPUUsageTimeID,
cpuConfig.MetricsConfigs[string(metrics.CPUUsageTimeID)].DisplayName,
"CPU usage, in seconds",
"s",
metrics.Sum,
[]string{stateLabel})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.CPUUsageTimeID, err)
}
cc.lastUsageTime = make(map[string]float64)
return &cc
}
func (cc *cpuCollector) recordLoad() {
if cc.mRunnableTaskCount == nil {
return
}
loadAvg, err := load.Avg()
if err != nil {
glog.Errorf("Failed to retrieve average CPU load: %v", err)
return
}
cc.mRunnableTaskCount.Record(map[string]string{}, loadAvg.Load1)
}
func (cc *cpuCollector) recordUsage() {
if cc.mUsageTime == nil {
return
}
// Set percpu=false to get aggregated usage from all CPUs.
timersStats, err := cpu.Times(false)
if err != nil {
glog.Errorf("Failed to retrieve CPU timers stat: %v", err)
return
}
timersStat := timersStats[0]
cc.mUsageTime.Record(map[string]string{stateLabel: "user"}, clockTick*timersStat.User-cc.lastUsageTime["user"])
cc.lastUsageTime["user"] = clockTick * timersStat.User
cc.mUsageTime.Record(map[string]string{stateLabel: "system"}, clockTick*timersStat.System-cc.lastUsageTime["system"])
cc.lastUsageTime["system"] = clockTick * timersStat.System
cc.mUsageTime.Record(map[string]string{stateLabel: "idle"}, clockTick*timersStat.Idle-cc.lastUsageTime["idle"])
cc.lastUsageTime["idle"] = clockTick * timersStat.Idle
cc.mUsageTime.Record(map[string]string{stateLabel: "nice"}, clockTick*timersStat.Nice-cc.lastUsageTime["nice"])
cc.lastUsageTime["nice"] = clockTick * timersStat.Nice
cc.mUsageTime.Record(map[string]string{stateLabel: "iowait"}, clockTick*timersStat.Iowait-cc.lastUsageTime["iowait"])
cc.lastUsageTime["iowait"] = clockTick * timersStat.Iowait
cc.mUsageTime.Record(map[string]string{stateLabel: "irq"}, clockTick*timersStat.Irq-cc.lastUsageTime["irq"])
cc.lastUsageTime["irq"] = clockTick * timersStat.Irq
cc.mUsageTime.Record(map[string]string{stateLabel: "softirq"}, clockTick*timersStat.Softirq-cc.lastUsageTime["softirq"])
cc.lastUsageTime["softirq"] = clockTick * timersStat.Softirq
cc.mUsageTime.Record(map[string]string{stateLabel: "steal"}, clockTick*timersStat.Steal-cc.lastUsageTime["steal"])
cc.lastUsageTime["steal"] = clockTick * timersStat.Steal
cc.mUsageTime.Record(map[string]string{stateLabel: "guest"}, clockTick*timersStat.Guest-cc.lastUsageTime["guest"])
cc.lastUsageTime["guest"] = clockTick * timersStat.Guest
cc.mUsageTime.Record(map[string]string{stateLabel: "guest_nice"}, clockTick*timersStat.GuestNice-cc.lastUsageTime["guest_nice"])
cc.lastUsageTime["guest_nice"] = clockTick * timersStat.GuestNice
}
func (cc *cpuCollector) collect() {
if cc == nil {
return
}
cc.recordLoad()
cc.recordUsage()
}
pkg/systemstatsmonitor/disk_collector.go
+185 -44
View File
@@ -29,18 +29,30 @@ import (
"k8s.io/node-problem-detector/pkg/util/metrics"
)
const deviceNameLabel = "device_name"
type diskCollector struct {
mIOTime *metrics.Int64Metric
mWeightedIO *metrics.Int64Metric
mAvgQueueLen *metrics.Float64Metric
mIOTime *metrics.Int64Metric
mWeightedIO *metrics.Int64Metric
mAvgQueueLen *metrics.Float64Metric
mOpsCount *metrics.Int64Metric
mMergedOpsCount *metrics.Int64Metric
mOpsBytes *metrics.Int64Metric
mOpsTime *metrics.Int64Metric
mBytesUsed *metrics.Int64Metric
config *ssmtypes.DiskStatsConfig
historyIOTime map[string]uint64
historyWeightedIO map[string]uint64
lastSampleTime time.Time
lastIOTime map[string]uint64
lastWeightedIO map[string]uint64
lastReadCount map[string]uint64
lastWriteCount map[string]uint64
lastMergedReadCount map[string]uint64
lastMergedWriteCount map[string]uint64
lastReadBytes map[string]uint64
lastWriteBytes map[string]uint64
lastReadTime map[string]uint64
lastWriteTime map[string]uint64
lastSampleTime time.Time
}
func NewDiskCollectorOrDie(diskConfig *ssmtypes.DiskStatsConfig) *diskCollector {
@@ -52,7 +64,7 @@ func NewDiskCollectorOrDie(diskConfig *ssmtypes.DiskStatsConfig) *diskCollector
dc.mIOTime, err = metrics.NewInt64Metric(
metrics.DiskIOTimeID,
diskConfig.MetricsConfigs[string(metrics.DiskIOTimeID)].DisplayName,
"The IO time spent on the disk",
"The IO time spent on the disk, in ms",
"ms",
metrics.Sum,
[]string{deviceNameLabel})
@@ -64,7 +76,7 @@ func NewDiskCollectorOrDie(diskConfig *ssmtypes.DiskStatsConfig) *diskCollector
dc.mWeightedIO, err = metrics.NewInt64Metric(
metrics.DiskWeightedIOID,
diskConfig.MetricsConfigs[string(metrics.DiskWeightedIOID)].DisplayName,
"The weighted IO on the disk",
"The weighted IO on the disk, in ms",
"ms",
metrics.Sum,
[]string{deviceNameLabel})
@@ -83,45 +95,86 @@ func NewDiskCollectorOrDie(diskConfig *ssmtypes.DiskStatsConfig) *diskCollector
glog.Fatalf("Error initializing metric for disk/avg_queue_len: %v", err)
}
dc.historyIOTime = make(map[string]uint64)
dc.historyWeightedIO = make(map[string]uint64)
dc.mOpsCount, err = metrics.NewInt64Metric(
metrics.DiskOpsCountID,
diskConfig.MetricsConfigs[string(metrics.DiskOpsCountID)].DisplayName,
"Disk operations count",
"1",
metrics.Sum,
[]string{deviceNameLabel, directionLabel})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.DiskOpsCountID, err)
}
dc.mMergedOpsCount, err = metrics.NewInt64Metric(
metrics.DiskMergedOpsCountID,
diskConfig.MetricsConfigs[string(metrics.DiskMergedOpsCountID)].DisplayName,
"Disk merged operations count",
"1",
metrics.Sum,
[]string{deviceNameLabel, directionLabel})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.DiskMergedOpsCountID, err)
}
dc.mOpsBytes, err = metrics.NewInt64Metric(
metrics.DiskOpsBytesID,
diskConfig.MetricsConfigs[string(metrics.DiskOpsBytesID)].DisplayName,
"Bytes transferred in disk operations",
"1",
metrics.Sum,
[]string{deviceNameLabel, directionLabel})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.DiskOpsBytesID, err)
}
dc.mOpsTime, err = metrics.NewInt64Metric(
metrics.DiskOpsTimeID,
diskConfig.MetricsConfigs[string(metrics.DiskOpsTimeID)].DisplayName,
"Time spent in disk operations, in ms",
"ms",
metrics.Sum,
[]string{deviceNameLabel, directionLabel})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.DiskOpsTimeID, err)
}
dc.mBytesUsed, err = metrics.NewInt64Metric(
metrics.DiskBytesUsedID,
diskConfig.MetricsConfigs[string(metrics.DiskBytesUsedID)].DisplayName,
"Disk bytes used, in Bytes",
"Byte",
metrics.LastValue,
[]string{deviceNameLabel, stateLabel})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.DiskBytesUsedID, err)
}
dc.lastIOTime = make(map[string]uint64)
dc.lastWeightedIO = make(map[string]uint64)
dc.lastReadCount = make(map[string]uint64)
dc.lastWriteCount = make(map[string]uint64)
dc.lastMergedReadCount = make(map[string]uint64)
dc.lastMergedWriteCount = make(map[string]uint64)
dc.lastReadBytes = make(map[string]uint64)
dc.lastWriteBytes = make(map[string]uint64)
dc.lastReadTime = make(map[string]uint64)
dc.lastWriteTime = make(map[string]uint64)
return &dc
}
func (dc *diskCollector) collect() {
if dc == nil {
return
}
blks := []string{}
if dc.config.IncludeRootBlk {
blks = append(blks, listRootBlockDevices(dc.config.LsblkTimeout)...)
}
if dc.config.IncludeAllAttachedBlk {
blks = append(blks, listAttachedBlockDevices()...)
}
ioCountersStats, err := disk.IOCounters(blks...)
if err != nil {
glog.Errorf("Failed to retrieve disk IO counters: %v", err)
return
}
sampleTime := time.Now()
func (dc *diskCollector) recordIOCounters(ioCountersStats map[string]disk.IOCountersStat, sampleTime time.Time) {
for deviceName, ioCountersStat := range ioCountersStats {
// Calculate average IO queue length since last measurement.
lastIOTime, historyExist := dc.historyIOTime[deviceName]
lastWeightedIO := dc.historyWeightedIO[deviceName]
lastSampleTime := dc.lastSampleTime
dc.historyIOTime[deviceName] = ioCountersStat.IoTime
dc.historyWeightedIO[deviceName] = ioCountersStat.WeightedIO
dc.lastSampleTime = sampleTime
// Attach label {"device_name": deviceName} to the metrics.
// Attach label {"device_name": deviceName} to the following metrics.
tags := map[string]string{deviceNameLabel: deviceName}
// Calculate average IO queue length since last measurement.
lastIOTime, historyExist := dc.lastIOTime[deviceName]
lastWeightedIO := dc.lastWeightedIO[deviceName]
dc.lastIOTime[deviceName] = ioCountersStat.IoTime
dc.lastWeightedIO[deviceName] = ioCountersStat.WeightedIO
if dc.mIOTime != nil {
dc.mIOTime.Record(tags, int64(ioCountersStat.IoTime-lastIOTime))
}
@@ -131,16 +184,104 @@ func (dc *diskCollector) collect() {
if historyExist {
avgQueueLen := float64(0.0)
if lastWeightedIO != ioCountersStat.WeightedIO {
diffSampleTimeMs := sampleTime.Sub(lastSampleTime).Seconds() * 1000
diffSampleTimeMs := sampleTime.Sub(dc.lastSampleTime).Seconds() * 1000
avgQueueLen = float64(ioCountersStat.WeightedIO-lastWeightedIO) / diffSampleTimeMs
}
if dc.mAvgQueueLen != nil {
dc.mAvgQueueLen.Record(tags, avgQueueLen)
}
}
// Attach label {"device_name": deviceName, "direction": "read"} to the following metrics.
tags = map[string]string{deviceNameLabel: deviceName, directionLabel: "read"}
if dc.mOpsCount != nil {
dc.mOpsCount.Record(tags, int64(ioCountersStat.ReadCount-dc.lastReadCount[deviceName]))
dc.lastReadCount[deviceName] = ioCountersStat.ReadCount
}
if dc.mMergedOpsCount != nil {
dc.mMergedOpsCount.Record(tags, int64(ioCountersStat.MergedReadCount-dc.lastMergedReadCount[deviceName]))
dc.lastMergedReadCount[deviceName] = ioCountersStat.MergedReadCount
}
if dc.mOpsBytes != nil {
dc.mOpsBytes.Record(tags, int64(ioCountersStat.ReadBytes-dc.lastReadBytes[deviceName]))
dc.lastReadBytes[deviceName] = ioCountersStat.ReadBytes
}
if dc.mOpsTime != nil {
dc.mOpsTime.Record(tags, int64(ioCountersStat.ReadTime-dc.lastReadTime[deviceName]))
dc.lastReadTime[deviceName] = ioCountersStat.ReadTime
}
// Attach label {"device_name": deviceName, "direction": "write"} to the following metrics.
tags = map[string]string{deviceNameLabel: deviceName, directionLabel: "write"}
if dc.mOpsCount != nil {
dc.mOpsCount.Record(tags, int64(ioCountersStat.WriteCount-dc.lastWriteCount[deviceName]))
dc.lastWriteCount[deviceName] = ioCountersStat.WriteCount
}
if dc.mMergedOpsCount != nil {
dc.mMergedOpsCount.Record(tags, int64(ioCountersStat.MergedWriteCount-dc.lastMergedWriteCount[deviceName]))
dc.lastMergedWriteCount[deviceName] = ioCountersStat.MergedWriteCount
}
if dc.mOpsBytes != nil {
dc.mOpsBytes.Record(tags, int64(ioCountersStat.WriteBytes-dc.lastWriteBytes[deviceName]))
dc.lastWriteBytes[deviceName] = ioCountersStat.WriteBytes
}
if dc.mOpsTime != nil {
dc.mOpsTime.Record(tags, int64(ioCountersStat.WriteTime-dc.lastWriteTime[deviceName]))
dc.lastWriteTime[deviceName] = ioCountersStat.WriteTime
}
}
}
func (dc *diskCollector) collect() {
if dc == nil {
return
}
// List available devices.
devices := []string{}
if dc.config.IncludeRootBlk {
devices = append(devices, listRootBlockDevices(dc.config.LsblkTimeout)...)
}
if dc.config.IncludeAllAttachedBlk {
devices = append(devices, listAttachedBlockDevices()...)
}
// Fetch metrics from /proc, /sys.
ioCountersStats, err := disk.IOCounters(devices...)
if err != nil {
glog.Errorf("Failed to retrieve disk IO counters: %v", err)
return
}
partitions, err := disk.Partitions(false)
if err != nil {
glog.Errorf("Failed to list disk partitions: %v", err)
return
}
sampleTime := time.Now()
defer func() { dc.lastSampleTime = sampleTime }()
// Record metrics regarding disk IO.
dc.recordIOCounters(ioCountersStats, sampleTime)
// Record metrics regarding disk space usage.
if dc.mBytesUsed == nil {
return
}
for _, partition := range partitions {
usageStat, err := disk.Usage(partition.Mountpoint)
if err != nil {
glog.Errorf("Failed to retrieve disk usage for %q: %v", partition.Mountpoint, err)
continue
}
deviceName := strings.TrimPrefix(partition.Device, "/dev/")
dc.mBytesUsed.Record(map[string]string{deviceNameLabel: deviceName, stateLabel: "free"}, int64(usageStat.Free))
dc.mBytesUsed.Record(map[string]string{deviceNameLabel: deviceName, stateLabel: "used"}, int64(usageStat.Used))
}
}
// listRootBlockDevices lists all block devices that's not a slave or holder.
func listRootBlockDevices(timeout time.Duration) []string {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
pkg/systemstatsmonitor/labels.go
+26
View File
@@ -0,0 +1,26 @@
/*
Copyright 2020 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package systemstatsmonitor
// deviceNameLabel labels the monitored disk device, e.g.: "sda", "sda1".
const deviceNameLabel = "device_name"
// directionLabel labels the direction of the disk operations, e.g.: "read", "write".
const directionLabel = "direction"
// stateLabel labels the state of disk/memory/cpu usage, e.g.: "free", "used".
const stateLabel = "state"
pkg/systemstatsmonitor/memory_collector.go
+143
View File
@@ -0,0 +1,143 @@
/*
Copyright 2020 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package systemstatsmonitor
import (
"github.com/golang/glog"
"github.com/prometheus/procfs"
ssmtypes "k8s.io/node-problem-detector/pkg/systemstatsmonitor/types"
"k8s.io/node-problem-detector/pkg/util/metrics"
)
type memoryCollector struct {
mBytesUsed *metrics.Int64Metric
mAnonymousUsed *metrics.Int64Metric
mPageCacheUsed *metrics.Int64Metric
mUnevictableUsed *metrics.Int64Metric
mDirtyUsed *metrics.Int64Metric
config *ssmtypes.MemoryStatsConfig
}
func NewMemoryCollectorOrDie(memoryConfig *ssmtypes.MemoryStatsConfig) *memoryCollector {
mc := memoryCollector{config: memoryConfig}
var err error
mc.mBytesUsed, err = metrics.NewInt64Metric(
metrics.MemoryBytesUsedID,
memoryConfig.MetricsConfigs[string(metrics.MemoryBytesUsedID)].DisplayName,
"Memory usage by each memory state, in Bytes. Summing values of all states yields the total memory on the node.",
"Byte",
metrics.LastValue,
[]string{stateLabel})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.MemoryBytesUsedID, err)
}
mc.mAnonymousUsed, err = metrics.NewInt64Metric(
metrics.MemoryAnonymousUsedID,
memoryConfig.MetricsConfigs[string(metrics.MemoryAnonymousUsedID)].DisplayName,
"Anonymous memory usage, in Bytes. Summing values of all states yields the total anonymous memory used.",
"Byte",
metrics.LastValue,
[]string{stateLabel})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.MemoryAnonymousUsedID, err)
}
mc.mPageCacheUsed, err = metrics.NewInt64Metric(
metrics.MemoryPageCacheUsedID,
memoryConfig.MetricsConfigs[string(metrics.MemoryPageCacheUsedID)].DisplayName,
"Page cache memory usage, in Bytes. Summing values of all states yields the total anonymous memory used.",
"Byte",
metrics.LastValue,
[]string{stateLabel})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.MemoryPageCacheUsedID, err)
}
mc.mUnevictableUsed, err = metrics.NewInt64Metric(
metrics.MemoryUnevictableUsedID,
memoryConfig.MetricsConfigs[string(metrics.MemoryUnevictableUsedID)].DisplayName,
"Unevictable memory usage, in Bytes",
"Byte",
metrics.LastValue,
[]string{})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.MemoryUnevictableUsedID, err)
}
mc.mDirtyUsed, err = metrics.NewInt64Metric(
metrics.MemoryDirtyUsedID,
memoryConfig.MetricsConfigs[string(metrics.MemoryDirtyUsedID)].DisplayName,
"Dirty pages usage, in Bytes. Dirty means the memory is waiting to be written back to disk, and writeback means the memory is actively being written back to disk.",
"Byte",
metrics.LastValue,
[]string{stateLabel})
if err != nil {
glog.Fatalf("Error initializing metric for %q: %v", metrics.MemoryDirtyUsedID, err)
}
return &mc
}
func (mc *memoryCollector) collect() {
if mc == nil {
return
}
proc, err := procfs.NewDefaultFS()
if err != nil {
glog.Errorf("Failed to find /proc mount point: %v", err)
return
}
meminfo, err := proc.Meminfo()
if err != nil {
glog.Errorf("Failed to retrieve memory stats: %v", err)
return
}
if mc.mBytesUsed != nil {
memUsed := meminfo.MemTotal - meminfo.MemFree - meminfo.Buffers - meminfo.Cached - meminfo.Slab
mc.mBytesUsed.Record(map[string]string{stateLabel: "free"}, int64(meminfo.MemFree))
mc.mBytesUsed.Record(map[string]string{stateLabel: "used"}, int64(memUsed))
mc.mBytesUsed.Record(map[string]string{stateLabel: "buffered"}, int64(meminfo.Buffers))
mc.mBytesUsed.Record(map[string]string{stateLabel: "cached"}, int64(meminfo.Cached))
mc.mBytesUsed.Record(map[string]string{stateLabel: "slab"}, int64(meminfo.Slab))
}
if mc.mDirtyUsed != nil {
mc.mDirtyUsed.Record(map[string]string{stateLabel: "dirty"}, int64(meminfo.Dirty))
mc.mDirtyUsed.Record(map[string]string{stateLabel: "writeback"}, int64(meminfo.Writeback))
}
if mc.mAnonymousUsed != nil {
mc.mAnonymousUsed.Record(map[string]string{stateLabel: "active"}, int64(meminfo.ActiveAnon))
mc.mAnonymousUsed.Record(map[string]string{stateLabel: "inactive"}, int64(meminfo.InactiveAnon))
}
if mc.mPageCacheUsed != nil {
mc.mPageCacheUsed.Record(map[string]string{stateLabel: "active"}, int64(meminfo.ActiveFile))
mc.mPageCacheUsed.Record(map[string]string{stateLabel: "inactive"}, int64(meminfo.InactiveFile))
}
if mc.mUnevictableUsed != nil {
mc.mUnevictableUsed.Record(map[string]string{}, int64(meminfo.Unevictable))
}
}
pkg/systemstatsmonitor/system_stats_monitor.go
+17 -5
View File
@@ -38,11 +38,13 @@ func init() {
}
type systemStatsMonitor struct {
configPath string
config ssmtypes.SystemStatsConfig
diskCollector *diskCollector
hostCollector *hostCollector
tomb *tomb.Tomb
configPath string
config ssmtypes.SystemStatsConfig
cpuCollector *cpuCollector
diskCollector *diskCollector
hostCollector *hostCollector
memoryCollector *memoryCollector
tomb *tomb.Tomb
}
// NewSystemStatsMonitorOrDie creates a system stats monitor.
@@ -72,12 +74,18 @@ func NewSystemStatsMonitorOrDie(configPath string) types.Monitor {
glog.Fatalf("Failed to validate %s configuration %+v: %v", ssm.configPath, ssm.config, err)
}
if len(ssm.config.CPUConfig.MetricsConfigs) > 0 {
ssm.cpuCollector = NewCPUCollectorOrDie(&ssm.config.CPUConfig)
}
if len(ssm.config.DiskConfig.MetricsConfigs) > 0 {
ssm.diskCollector = NewDiskCollectorOrDie(&ssm.config.DiskConfig)
}
if len(ssm.config.HostConfig.MetricsConfigs) > 0 {
ssm.hostCollector = NewHostCollectorOrDie(&ssm.config.HostConfig)
}
if len(ssm.config.MemoryConfig.MetricsConfigs) > 0 {
ssm.memoryCollector = NewMemoryCollectorOrDie(&ssm.config.MemoryConfig)
}
return &ssm
}
@@ -98,15 +106,19 @@ func (ssm *systemStatsMonitor) monitorLoop() {
glog.Infof("System stats monitor stopped: %s", ssm.configPath)
return
default:
ssm.cpuCollector.collect()
ssm.diskCollector.collect()
ssm.hostCollector.collect()
ssm.memoryCollector.collect()
}
for {
select {
case <-runTicker.C:
ssm.cpuCollector.collect()
ssm.diskCollector.collect()
ssm.hostCollector.collect()
ssm.memoryCollector.collect()
case <-ssm.tomb.Stopping():
glog.Infof("System stats monitor stopped: %s", ssm.configPath)
return
pkg/systemstatsmonitor/types/config.go
+14 -4
View File
@@ -30,6 +30,10 @@ type MetricConfig struct {
DisplayName string `json:"displayName"`
}
type CPUStatsConfig struct {
MetricsConfigs map[string]MetricConfig `json:"metricsConfigs"`
}
type DiskStatsConfig struct {
MetricsConfigs map[string]MetricConfig `json:"metricsConfigs"`
IncludeRootBlk bool `json:"includeRootBlk"`
@@ -42,11 +46,17 @@ type HostStatsConfig struct {
MetricsConfigs map[string]MetricConfig `json:"metricsConfigs"`
}
type MemoryStatsConfig struct {
MetricsConfigs map[string]MetricConfig `json:"metricsConfigs"`
}
type SystemStatsConfig struct {
DiskConfig DiskStatsConfig `json:"disk"`
HostConfig HostStatsConfig `json:"host"`
InvokeIntervalString string `json:"invokeInterval"`
InvokeInterval time.Duration `json:"-"`
CPUConfig CPUStatsConfig `json:"cpu"`
DiskConfig DiskStatsConfig `json:"disk"`
HostConfig HostStatsConfig `json:"host"`
MemoryConfig MemoryStatsConfig `json:"memory"`
InvokeIntervalString string `json:"invokeInterval"`
InvokeInterval time.Duration `json:"-"`
}
// ApplyConfiguration applies default configurations.
pkg/util/metrics/metric.go
+18 -6
View File
@@ -20,12 +20,24 @@ import (
)
const (
ProblemCounterID MetricID = "problem_counter"
ProblemGaugeID MetricID = "problem_gauge"
DiskIOTimeID MetricID = "disk/io_time"
DiskWeightedIOID MetricID = "disk/weighted_io"
DiskAvgQueueLenID MetricID = "disk/avg_queue_len"
HostUptimeID MetricID = "host/uptime"
CPURunnableTaskCountID MetricID = "cpu/runnable_task_count"
CPUUsageTimeID MetricID = "cpu/usage_time"
ProblemCounterID MetricID = "problem_counter"
ProblemGaugeID MetricID = "problem_gauge"
DiskIOTimeID MetricID = "disk/io_time"
DiskWeightedIOID MetricID = "disk/weighted_io"
DiskAvgQueueLenID MetricID = "disk/avg_queue_len"
DiskOpsCountID MetricID = "disk/operation_count"
DiskMergedOpsCountID MetricID = "disk/merged_operation_count"
DiskOpsBytesID MetricID = "disk/operation_bytes_count"
DiskOpsTimeID MetricID = "disk/operation_time"
DiskBytesUsedID MetricID = "disk/bytes_used"
HostUptimeID MetricID = "host/uptime"
MemoryBytesUsedID MetricID = "memory/bytes_used"
MemoryAnonymousUsedID MetricID = "memory/anonymous_used"
MemoryPageCacheUsedID MetricID = "memory/page_cache_used"
MemoryUnevictableUsedID MetricID = "memory/unevictable_used"
MemoryDirtyUsedID MetricID = "memory/dirty_used"
)
var MetricMap MetricMapping