From 4944ac3e48a7008b5e7bd93798575a6d4853fff9 Mon Sep 17 00:00:00 2001 From: Xuewei Zhang Date: Thu, 27 Jun 2019 11:27:05 -0700 Subject: [PATCH] Implement host collector as part of system-stats-monitor Host collector report three things today: 1. Host OS uptime (in seconds) 2. Host kernel version (as a metric label) 3. Host OS version (as a metric label) --- config/system-stats-monitor.json | 7 ++ pkg/systemstatsmonitor/disk_collector.go | 80 +++++++++++------ pkg/systemstatsmonitor/host_collector.go | 90 +++++++++++++++++++ .../system_stats_monitor.go | 7 +- pkg/systemstatsmonitor/types/config.go | 5 ++ pkg/util/helpers.go | 35 ++++++++ pkg/util/helpers_test.go | 62 +++++++++++++ pkg/util/testdata/os-release-cos | 11 +++ pkg/util/testdata/os-release-debian | 8 ++ pkg/util/testdata/os-release-empty | 0 pkg/util/testdata/os-release-ubuntu | 11 +++ pkg/util/testdata/os-release-unknown | 1 + 12 files changed, 289 insertions(+), 28 deletions(-) create mode 100644 pkg/systemstatsmonitor/host_collector.go create mode 100644 pkg/util/testdata/os-release-cos create mode 100644 pkg/util/testdata/os-release-debian create mode 100644 pkg/util/testdata/os-release-empty create mode 100644 pkg/util/testdata/os-release-ubuntu create mode 100644 pkg/util/testdata/os-release-unknown diff --git a/config/system-stats-monitor.json b/config/system-stats-monitor.json index d9bfd603..88faa162 100644 --- a/config/system-stats-monitor.json +++ b/config/system-stats-monitor.json @@ -15,5 +15,12 @@ "includeAllAttachedBlk": true, "lsblkTimeout": "5s" }, + "host": { + "metricsConfigs": { + "host/uptime": { + "displayName": "host/uptime" + } + } + }, "invokeInterval": "60s" } diff --git a/pkg/systemstatsmonitor/disk_collector.go b/pkg/systemstatsmonitor/disk_collector.go index fce560d7..fbf9c844 100644 --- a/pkg/systemstatsmonitor/disk_collector.go +++ b/pkg/systemstatsmonitor/disk_collector.go @@ -46,28 +46,39 @@ type diskCollector struct { func NewDiskCollectorOrDie(diskConfig *ssmtypes.DiskStatsConfig) *diskCollector { dc := diskCollector{config: diskConfig} - dc.keyDevice, _ = tag.NewKey("device") - dc.mIOTime = metrics.NewInt64Metric( - diskConfig.MetricsConfigs["disk/io_time"].DisplayName, - "The IO time spent on the disk", - "second", - view.LastValue(), - []tag.Key{dc.keyDevice}) + var err error + dc.keyDevice, err = tag.NewKey("device") + if err != nil { + glog.Fatalf("Failed to create device tag during initializing disk collector: %v", err) + } - dc.mWeightedIO = metrics.NewInt64Metric( - diskConfig.MetricsConfigs["disk/weighted_io"].DisplayName, - "The weighted IO on the disk", - "second", - view.LastValue(), - []tag.Key{dc.keyDevice}) + if diskConfig.MetricsConfigs["disk/io_time"].DisplayName != "" { + dc.mIOTime = metrics.NewInt64Metric( + diskConfig.MetricsConfigs["disk/io_time"].DisplayName, + "The IO time spent on the disk", + "second", + view.LastValue(), + []tag.Key{dc.keyDevice}) + } - dc.mAvgQueueLen = metrics.NewFloat64Metric( - diskConfig.MetricsConfigs["disk/avg_queue_len"].DisplayName, - "The average queue length on the disk", - "second", - view.LastValue(), - []tag.Key{dc.keyDevice}) + if diskConfig.MetricsConfigs["disk/weighted_io"].DisplayName != "" { + dc.mWeightedIO = metrics.NewInt64Metric( + diskConfig.MetricsConfigs["disk/weighted_io"].DisplayName, + "The weighted IO on the disk", + "second", + view.LastValue(), + []tag.Key{dc.keyDevice}) + } + + if diskConfig.MetricsConfigs["disk/avg_queue_len"].DisplayName != "" { + dc.mAvgQueueLen = metrics.NewFloat64Metric( + diskConfig.MetricsConfigs["disk/avg_queue_len"].DisplayName, + "The average queue length on the disk", + "second", + view.LastValue(), + []tag.Key{dc.keyDevice}) + } dc.historyIOTime = make(map[string]uint64) dc.historyWeightedIO = make(map[string]uint64) @@ -88,7 +99,11 @@ func (dc *diskCollector) collect() { blks = append(blks, listAttachedBlockDevices()...) } - ioCountersStats, _ := disk.IOCounters(blks...) + ioCountersStats, err := disk.IOCounters(blks...) + if err != nil { + glog.Errorf("Failed to retrieve disk IO counters: %v", err) + return + } for deviceName, ioCountersStat := range ioCountersStats { // Calculate average IO queue length since last measurement. @@ -98,21 +113,26 @@ func (dc *diskCollector) collect() { dc.historyIOTime[deviceName] = ioCountersStat.IoTime dc.historyWeightedIO[deviceName] = ioCountersStat.WeightedIO - avg_queue_len := float64(0.0) + avgQueueLen := float64(0.0) if lastIOTime != ioCountersStat.IoTime { - avg_queue_len = float64(ioCountersStat.WeightedIO-lastWeightedIO) / float64(ioCountersStat.IoTime-lastIOTime) + avgQueueLen = float64(ioCountersStat.WeightedIO-lastWeightedIO) / float64(ioCountersStat.IoTime-lastIOTime) } // Attach label {"device": deviceName} to the metrics. - device_ctx, _ := tag.New(context.Background(), tag.Upsert(dc.keyDevice, deviceName)) + deviceCtx, err := tag.New(context.Background(), tag.Upsert(dc.keyDevice, deviceName)) + if err != nil { + glog.Errorf("Failed to create context with device tag: %v", err) + deviceCtx = context.Background() + } + if dc.mIOTime != nil { - stats.Record(device_ctx, dc.mIOTime.M(int64(ioCountersStat.IoTime))) + stats.Record(deviceCtx, dc.mIOTime.M(int64(ioCountersStat.IoTime))) } if dc.mWeightedIO != nil { - stats.Record(device_ctx, dc.mWeightedIO.M(int64(ioCountersStat.WeightedIO))) + stats.Record(deviceCtx, dc.mWeightedIO.M(int64(ioCountersStat.WeightedIO))) } if dc.mAvgQueueLen != nil { - stats.Record(device_ctx, dc.mAvgQueueLen.M(avg_queue_len)) + stats.Record(deviceCtx, dc.mAvgQueueLen.M(avgQueueLen)) } } } @@ -135,8 +155,14 @@ func listRootBlockDevices(timeout time.Duration) []string { // listAttachedBlockDevices lists all currently attached block devices. func listAttachedBlockDevices() []string { - partitions, _ := disk.Partitions(false) blks := []string{} + + partitions, err := disk.Partitions(false) + if err != nil { + glog.Errorf("Failed to retrieve the list of disk partitions: %v", err) + return blks + } + for _, partition := range partitions { blks = append(blks, partition.Device) } diff --git a/pkg/systemstatsmonitor/host_collector.go b/pkg/systemstatsmonitor/host_collector.go new file mode 100644 index 00000000..fdbcaa1e --- /dev/null +++ b/pkg/systemstatsmonitor/host_collector.go @@ -0,0 +1,90 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package systemstatsmonitor + +import ( + "context" + + "github.com/golang/glog" + "github.com/shirou/gopsutil/host" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + + ssmtypes "k8s.io/node-problem-detector/pkg/systemstatsmonitor/types" + "k8s.io/node-problem-detector/pkg/util" + "k8s.io/node-problem-detector/pkg/util/metrics" +) + +type hostCollector struct { + tags []tag.Mutator + uptime *stats.Int64Measure +} + +func NewHostCollectorOrDie(hostConfig *ssmtypes.HostStatsConfig) *hostCollector { + hc := hostCollector{} + + keyKernelVersion, err := tag.NewKey("kernel_version") + if err != nil { + glog.Fatalf("Failed to create kernel_version tag during initializing host collector: %v", err) + } + kernelVersion, err := host.KernelVersion() + if err != nil { + glog.Fatalf("Failed to retrieve kernel version: %v", err) + } + hc.tags = append(hc.tags, tag.Upsert(keyKernelVersion, kernelVersion)) + + keyOSVersion, err := tag.NewKey("os_version") + if err != nil { + glog.Fatalf("Failed to create os_version tag during initializing host collector: %v", err) + } + osVersion, err := util.GetOSVersion() + if err != nil { + glog.Fatalf("Failed to retrieve OS version: %v", err) + } + hc.tags = append(hc.tags, tag.Upsert(keyOSVersion, osVersion)) + + if hostConfig.MetricsConfigs["host/uptime"].DisplayName != "" { + hc.uptime = metrics.NewInt64Metric( + hostConfig.MetricsConfigs["host/uptime"].DisplayName, + "The uptime of the operating system", + "second", + view.LastValue(), + []tag.Key{keyKernelVersion, keyOSVersion}) + } + + return &hc +} + +func (hc *hostCollector) collect() { + if hc == nil { + return + } + + uptime, err := host.Uptime() + if err != nil { + glog.Errorf("Failed to retrieve uptime of the host: %v", err) + return + } + + if hc.uptime != nil { + err := stats.RecordWithTags(context.Background(), hc.tags, hc.uptime.M(int64(uptime))) + if err != nil { + glog.Errorf("Failed to record current uptime (%d seconds) of the host: %v", uptime, err) + } + } +} diff --git a/pkg/systemstatsmonitor/system_stats_monitor.go b/pkg/systemstatsmonitor/system_stats_monitor.go index c14d2b8e..d03b4ae0 100644 --- a/pkg/systemstatsmonitor/system_stats_monitor.go +++ b/pkg/systemstatsmonitor/system_stats_monitor.go @@ -40,6 +40,7 @@ func init() { type systemStatsMonitor struct { config ssmtypes.SystemStatsConfig diskCollector *diskCollector + hostCollector *hostCollector tomb *tomb.Tomb } @@ -69,10 +70,12 @@ func NewSystemStatsMonitorOrDie(configPath string) types.Monitor { glog.Fatalf("Failed to validate configuration %+v: %v", ssm.config, err) } - // Initialize diskCollector if needed. if len(ssm.config.DiskConfig.MetricsConfigs) > 0 { ssm.diskCollector = NewDiskCollectorOrDie(&ssm.config.DiskConfig) } + if len(ssm.config.HostConfig.MetricsConfigs) > 0 { + ssm.hostCollector = NewHostCollectorOrDie(&ssm.config.HostConfig) + } return &ssm } @@ -94,12 +97,14 @@ func (ssm *systemStatsMonitor) monitorLoop() { return default: ssm.diskCollector.collect() + ssm.hostCollector.collect() } for { select { case <-runTicker.C: ssm.diskCollector.collect() + ssm.hostCollector.collect() case <-ssm.tomb.Stopping(): glog.Infof("System stats monitor stopped") return diff --git a/pkg/systemstatsmonitor/types/config.go b/pkg/systemstatsmonitor/types/config.go index 4ffa9623..c10fd2c2 100644 --- a/pkg/systemstatsmonitor/types/config.go +++ b/pkg/systemstatsmonitor/types/config.go @@ -38,8 +38,13 @@ type DiskStatsConfig struct { LsblkTimeout time.Duration `json:"-"` } +type HostStatsConfig struct { + MetricsConfigs map[string]MetricConfig `json:"metricsConfigs"` +} + type SystemStatsConfig struct { DiskConfig DiskStatsConfig `json:"disk"` + HostConfig HostStatsConfig `json:"host"` InvokeIntervalString string `json:"invokeInterval"` InvokeInterval time.Duration `json:"-"` } diff --git a/pkg/util/helpers.go b/pkg/util/helpers.go index 7b659d27..35045def 100644 --- a/pkg/util/helpers.go +++ b/pkg/util/helpers.go @@ -20,9 +20,13 @@ import ( "syscall" "time" + "github.com/cobaugh/osrelease" + "k8s.io/node-problem-detector/pkg/types" ) +var osReleasePath = "/etc/os-release" + // GenerateConditionChangeEvent generates an event for condition change. func GenerateConditionChangeEvent(t string, status types.ConditionStatus, reason string, timestamp time.Time) types.Event { return types.Event{ @@ -70,3 +74,34 @@ func GetStartTime(now time.Time, uptimeDuration time.Duration, lookbackStr strin return startTime, nil } + +// GetOSVersion retrieves the version of the current operating system. +// For example: "cos 77-12293.0.0", "ubuntu 16.04.6 LTS (Xenial Xerus)". +func GetOSVersion() (string, error) { + osReleaseMap, err := osrelease.ReadFile(osReleasePath) + if err != nil { + return "", err + } + switch osReleaseMap["ID"] { + case "cos": + return getCOSVersion(osReleaseMap), nil + case "debian": + return getDebianVersion(osReleaseMap), nil + case "ubuntu": + return getDebianVersion(osReleaseMap), nil + default: + return "", fmt.Errorf("Unsupported ID in /etc/os-release: %q", osReleaseMap["ID"]) + } +} + +func getCOSVersion(osReleaseMap map[string]string) string { + // /etc/os-release syntax for COS is defined here: + // https://chromium.git.corp.google.com/chromiumos/docs/+/8edec95a297edfd8f1290f0f03a8aa35795b516b/os_config.md + return fmt.Sprintf("%s %s-%s", osReleaseMap["ID"], osReleaseMap["VERSION"], osReleaseMap["BUILD_ID"]) +} + +func getDebianVersion(osReleaseMap map[string]string) string { + // /etc/os-release syntax for Debian is defined here: + // https://manpages.debian.org/testing/systemd/os-release.5.en.html + return fmt.Sprintf("%s %s", osReleaseMap["ID"], osReleaseMap["VERSION"]) +} diff --git a/pkg/util/helpers_test.go b/pkg/util/helpers_test.go index a664e5e7..32b3ff5e 100644 --- a/pkg/util/helpers_test.go +++ b/pkg/util/helpers_test.go @@ -135,3 +135,65 @@ func TestGetStartTime(t *testing.T) { }) } } + +func TestGetOSVersion(t *testing.T) { + testCases := []struct { + name string + fakeOSReleasePath string + expectedOSVersion string + expectErr bool + }{ + { + name: "COS", + fakeOSReleasePath: "testdata/os-release-cos", + expectedOSVersion: "cos 77-12293.0.0", + expectErr: false, + }, + { + name: "Debian", + fakeOSReleasePath: "testdata/os-release-debian", + expectedOSVersion: "debian 9 (stretch)", + expectErr: false, + }, + { + name: "Ubuntu", + fakeOSReleasePath: "testdata/os-release-ubuntu", + expectedOSVersion: "ubuntu 16.04.6 LTS (Xenial Xerus)", + expectErr: false, + }, + { + name: "Unknown", + fakeOSReleasePath: "testdata/os-release-unknown", + expectedOSVersion: "", + expectErr: true, + }, + { + name: "Empty", + fakeOSReleasePath: "testdata/os-release-empty", + expectedOSVersion: "", + expectErr: true, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + originalOSReleasePath := osReleasePath + defer func() { + osReleasePath = originalOSReleasePath + }() + + osReleasePath = test.fakeOSReleasePath + osVersion, err := GetOSVersion() + + if test.expectErr && err == nil { + t.Errorf("Expect to get error, but got no returned error.") + } + if !test.expectErr && err != nil { + t.Errorf("Expect to get no error, but got returned error: %v", err) + } + if !test.expectErr && osVersion != test.expectedOSVersion { + t.Errorf("Wanted: %+v. \nGot: %+v", test.expectedOSVersion, osVersion) + } + }) + } +} diff --git a/pkg/util/testdata/os-release-cos b/pkg/util/testdata/os-release-cos new file mode 100644 index 00000000..425b7066 --- /dev/null +++ b/pkg/util/testdata/os-release-cos @@ -0,0 +1,11 @@ +BUILD_ID=12293.0.0 +NAME="Container-Optimized OS" +KERNEL_COMMIT_ID=337e5a1ca410d2b7ab6e53a87e51b60fddab0869 +GOOGLE_CRASH_ID=Lakitu +VERSION_ID=77 +BUG_REPORT_URL="https://cloud.google.com/container-optimized-os/docs/resources/support-policy#contact_us" +PRETTY_NAME="Container-Optimized OS from Google" +VERSION=77 +GOOGLE_METRICS_PRODUCT_ID=26 +HOME_URL="https://cloud.google.com/container-optimized-os/docs" +ID=cos diff --git a/pkg/util/testdata/os-release-debian b/pkg/util/testdata/os-release-debian new file mode 100644 index 00000000..ea56ca0d --- /dev/null +++ b/pkg/util/testdata/os-release-debian @@ -0,0 +1,8 @@ +PRETTY_NAME="Debian GNU/Linux 9 (stretch)" +NAME="Debian GNU/Linux" +VERSION_ID="9" +VERSION="9 (stretch)" +ID=debian +HOME_URL="https://www.debian.org/" +SUPPORT_URL="https://www.debian.org/support" +BUG_REPORT_URL="https://bugs.debian.org/" diff --git a/pkg/util/testdata/os-release-empty b/pkg/util/testdata/os-release-empty new file mode 100644 index 00000000..e69de29b diff --git a/pkg/util/testdata/os-release-ubuntu b/pkg/util/testdata/os-release-ubuntu new file mode 100644 index 00000000..d748d556 --- /dev/null +++ b/pkg/util/testdata/os-release-ubuntu @@ -0,0 +1,11 @@ +NAME="Ubuntu" +VERSION="16.04.6 LTS (Xenial Xerus)" +ID=ubuntu +ID_LIKE=debian +PRETTY_NAME="Ubuntu 16.04.6 LTS" +VERSION_ID="16.04" +HOME_URL="http://www.ubuntu.com/" +SUPPORT_URL="http://help.ubuntu.com/" +BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/" +VERSION_CODENAME=xenial +UBUNTU_CODENAME=xenial diff --git a/pkg/util/testdata/os-release-unknown b/pkg/util/testdata/os-release-unknown new file mode 100644 index 00000000..28b51fee --- /dev/null +++ b/pkg/util/testdata/os-release-unknown @@ -0,0 +1 @@ +ID=foo-operating-system