github/node-problem-detector
1
0
Fork 0
mirror of https://github.com/kubernetes/node-problem-detector.git synced 2026-08-20 12:46:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Report metrics from system-log-monitor

Browse Source
This commit is contained in:
Xuewei Zhang
2019-07-12 14:38:21 -07:00
parent 30babe906e
commit fbebcf311b
18 changed files with 1608 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/systemstatsmonitor/disk_collector.go
+30 -41
View File
@@ -24,19 +24,15 @@ import (
"github.com/golang/glog"
"github.com/shirou/gopsutil/disk"
"go.opencensus.io/stats"
"go.opencensus.io/stats/view"
"go.opencensus.io/tag"
ssmtypes "k8s.io/node-problem-detector/pkg/systemstatsmonitor/types"
"k8s.io/node-problem-detector/pkg/util/metrics"
)
type diskCollector struct {
keyDevice tag.Key
mIOTime *stats.Int64Measure
mWeightedIO *stats.Int64Measure
mAvgQueueLen *stats.Float64Measure
mIOTime *metrics.Int64Metric
mWeightedIO *metrics.Int64Metric
mAvgQueueLen *metrics.Float64Metric
config *ssmtypes.DiskStatsConfig
@@ -48,36 +44,34 @@ func NewDiskCollectorOrDie(diskConfig *ssmtypes.DiskStatsConfig) *diskCollector
dc := diskCollector{config: diskConfig}
var err error
dc.keyDevice, err = tag.NewKey("device")
dc.mIOTime, err = metrics.NewInt64Metric(
diskConfig.MetricsConfigs["disk/io_time"].DisplayName,
"The IO time spent on the disk",
"second",
metrics.LastValue,
[]string{"device"})
if err != nil {
glog.Fatalf("Failed to create device tag during initializing disk collector: %v", err)
glog.Fatalf("Error initializing metric for disk/io_time: %v", err)
}
if diskConfig.MetricsConfigs["disk/io_time"].DisplayName != "" {
dc.mIOTime = metrics.NewInt64Metric(
diskConfig.MetricsConfigs["disk/io_time"].DisplayName,
"The IO time spent on the disk",
"second",
view.LastValue(),
[]tag.Key{dc.keyDevice})
dc.mWeightedIO, err = metrics.NewInt64Metric(
diskConfig.MetricsConfigs["disk/weighted_io"].DisplayName,
"The weighted IO on the disk",
"second",
metrics.LastValue,
[]string{"device"})
if err != nil {
glog.Fatalf("Error initializing metric for disk/weighted_io: %v", err)
}
if diskConfig.MetricsConfigs["disk/weighted_io"].DisplayName != "" {
dc.mWeightedIO = metrics.NewInt64Metric(
diskConfig.MetricsConfigs["disk/weighted_io"].DisplayName,
"The weighted IO on the disk",
"second",
view.LastValue(),
[]tag.Key{dc.keyDevice})
}
if diskConfig.MetricsConfigs["disk/avg_queue_len"].DisplayName != "" {
dc.mAvgQueueLen = metrics.NewFloat64Metric(
diskConfig.MetricsConfigs["disk/avg_queue_len"].DisplayName,
"The average queue length on the disk",
"second",
view.LastValue(),
[]tag.Key{dc.keyDevice})
dc.mAvgQueueLen, err = metrics.NewFloat64Metric(
diskConfig.MetricsConfigs["disk/avg_queue_len"].DisplayName,
"The average queue length on the disk",
"second",
metrics.LastValue,
[]string{"device"})
if err != nil {
glog.Fatalf("Error initializing metric for disk/avg_queue_len: %v", err)
}
dc.historyIOTime = make(map[string]uint64)
@@ -119,20 +113,15 @@ func (dc *diskCollector) collect() {
}
// Attach label {"device": deviceName} to the metrics.
deviceCtx, err := tag.New(context.Background(), tag.Upsert(dc.keyDevice, deviceName))
if err != nil {
glog.Errorf("Failed to create context with device tag: %v", err)
deviceCtx = context.Background()
}
tags := map[string]string{"device": deviceName}
if dc.mIOTime != nil {
stats.Record(deviceCtx, dc.mIOTime.M(int64(ioCountersStat.IoTime)))
dc.mIOTime.Record(tags, int64(ioCountersStat.IoTime))
}
if dc.mWeightedIO != nil {
stats.Record(deviceCtx, dc.mWeightedIO.M(int64(ioCountersStat.WeightedIO)))
dc.mWeightedIO.Record(tags, int64(ioCountersStat.WeightedIO))
}
if dc.mAvgQueueLen != nil {
stats.Record(deviceCtx, dc.mAvgQueueLen.M(avgQueueLen))
dc.mAvgQueueLen.Record(tags, avgQueueLen)
}
}
}
pkg/systemstatsmonitor/host_collector.go
+12 -25
View File
@@ -17,13 +17,8 @@ limitations under the License.
package systemstatsmonitor
import (
"context"
"github.com/golang/glog"
"github.com/shirou/gopsutil/host"
"go.opencensus.io/stats"
"go.opencensus.io/stats/view"
"go.opencensus.io/tag"
ssmtypes "k8s.io/node-problem-detector/pkg/systemstatsmonitor/types"
"k8s.io/node-problem-detector/pkg/util"
@@ -31,40 +26,35 @@ import (
)
type hostCollector struct {
tags []tag.Mutator
uptime *stats.Int64Measure
tags map[string]string
uptime *metrics.Int64Metric
}
func NewHostCollectorOrDie(hostConfig *ssmtypes.HostStatsConfig) *hostCollector {
hc := hostCollector{}
hc := hostCollector{map[string]string{}, nil}
keyKernelVersion, err := tag.NewKey("kernel_version")
if err != nil {
glog.Fatalf("Failed to create kernel_version tag during initializing host collector: %v", err)
}
kernelVersion, err := host.KernelVersion()
if err != nil {
glog.Fatalf("Failed to retrieve kernel version: %v", err)
}
hc.tags = append(hc.tags, tag.Upsert(keyKernelVersion, kernelVersion))
hc.tags["kernel_version"] = kernelVersion
keyOSVersion, err := tag.NewKey("os_version")
if err != nil {
glog.Fatalf("Failed to create os_version tag during initializing host collector: %v", err)
}
osVersion, err := util.GetOSVersion()
if err != nil {
glog.Fatalf("Failed to retrieve OS version: %v", err)
}
hc.tags = append(hc.tags, tag.Upsert(keyOSVersion, osVersion))
hc.tags["os_version"] = osVersion
if hostConfig.MetricsConfigs["host/uptime"].DisplayName != "" {
hc.uptime = metrics.NewInt64Metric(
hc.uptime, err = metrics.NewInt64Metric(
hostConfig.MetricsConfigs["host/uptime"].DisplayName,
"The uptime of the operating system",
"second",
view.LastValue(),
[]tag.Key{keyKernelVersion, keyOSVersion})
metrics.LastValue,
[]string{"kernel_version", "os_version"})
if err != nil {
glog.Fatalf("Error initializing metric for host/uptime: %v", err)
}
}
return &hc
@@ -82,9 +72,6 @@ func (hc *hostCollector) collect() {
}
if hc.uptime != nil {
err := stats.RecordWithTags(context.Background(), hc.tags, hc.uptime.M(int64(uptime)))
if err != nil {
glog.Errorf("Failed to record current uptime (%d seconds) of the host: %v", uptime, err)
}
hc.uptime.Record(hc.tags, int64(uptime))
}
}