From 3d10c892a25b263c5872def091cb144c9ec4f96a Mon Sep 17 00:00:00 2001 From: Alex Wong Date: Wed, 11 Dec 2019 11:13:43 +0800 Subject: [PATCH 1/2] Ignore first collected disk stats to prevent metric distortion --- pkg/systemstatsmonitor/disk_collector.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/systemstatsmonitor/disk_collector.go b/pkg/systemstatsmonitor/disk_collector.go index 6bb07b60..bcc66b0c 100644 --- a/pkg/systemstatsmonitor/disk_collector.go +++ b/pkg/systemstatsmonitor/disk_collector.go @@ -109,12 +109,16 @@ func (dc *diskCollector) collect() { for deviceName, ioCountersStat := range ioCountersStats { // Calculate average IO queue length since last measurement. - lastIOTime := dc.historyIOTime[deviceName] + lastIOTime, historyExist := dc.historyIOTime[deviceName] lastWeightedIO := dc.historyWeightedIO[deviceName] dc.historyIOTime[deviceName] = ioCountersStat.IoTime dc.historyWeightedIO[deviceName] = ioCountersStat.WeightedIO + if !historyExist { + // Ignore first collected stats. + return + } avgQueueLen := float64(0.0) if lastIOTime != ioCountersStat.IoTime { avgQueueLen = float64(ioCountersStat.WeightedIO-lastWeightedIO) / float64(ioCountersStat.IoTime-lastIOTime) From 5a4ac811866ad501da781fe32887732ef1ac0228 Mon Sep 17 00:00:00 2001 From: Alex Wong Date: Thu, 12 Dec 2019 14:39:29 +0800 Subject: [PATCH 2/2] Only disk_avg_queue_len is distorted on first collection --- pkg/systemstatsmonitor/disk_collector.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/pkg/systemstatsmonitor/disk_collector.go b/pkg/systemstatsmonitor/disk_collector.go index bcc66b0c..a4b3715f 100644 --- a/pkg/systemstatsmonitor/disk_collector.go +++ b/pkg/systemstatsmonitor/disk_collector.go @@ -115,15 +115,6 @@ func (dc *diskCollector) collect() { dc.historyIOTime[deviceName] = ioCountersStat.IoTime dc.historyWeightedIO[deviceName] = ioCountersStat.WeightedIO - if !historyExist { - // Ignore first collected stats. - return - } - avgQueueLen := float64(0.0) - if lastIOTime != ioCountersStat.IoTime { - avgQueueLen = float64(ioCountersStat.WeightedIO-lastWeightedIO) / float64(ioCountersStat.IoTime-lastIOTime) - } - // Attach label {"device_name": deviceName} to the metrics. tags := map[string]string{deviceNameLabel: deviceName} if dc.mIOTime != nil { @@ -132,8 +123,14 @@ func (dc *diskCollector) collect() { if dc.mWeightedIO != nil { dc.mWeightedIO.Record(tags, int64(ioCountersStat.WeightedIO-lastWeightedIO)) } - if dc.mAvgQueueLen != nil { - dc.mAvgQueueLen.Record(tags, avgQueueLen) + if historyExist { + avgQueueLen := float64(0.0) + if lastIOTime != ioCountersStat.IoTime { + avgQueueLen = float64(ioCountersStat.WeightedIO-lastWeightedIO) / float64(ioCountersStat.IoTime-lastIOTime) + } + if dc.mAvgQueueLen != nil { + dc.mAvgQueueLen.Record(tags, avgQueueLen) + } } } }