From 0f0e5eff0fd17588e4086123b8aac298f0500cad Mon Sep 17 00:00:00 2001 From: Xuewei Zhang Date: Mon, 9 Sep 2019 19:12:31 -0700 Subject: [PATCH] Adding stackdriver exporter --- .travis.yml | 2 + Makefile | 2 +- README.md | 41 +++- .../exporterplugins/default_plugin.go | 20 ++ .../stackdriver_exporter_plugin.go | 25 +++ .../node_problem_detector.go | 19 +- cmd/options/options.go | 6 + config/exporter/stackdriver-exporter.json | 8 + .../node-problem-detector-metric-only.service | 5 +- pkg/exporters/register.go | 63 ++++++ pkg/exporters/register_test.go | 69 +++++++ pkg/exporters/stackdriver/config/config.go | 56 ++++++ .../stackdriver/config/config_test.go | 107 ++++++++++ pkg/exporters/stackdriver/gce/type.go | 67 ++++++ .../stackdriver/stackdriver_exporter.go | 190 ++++++++++++++++++ .../stackdriver/stackdriver_exporter_test.go | 33 +++ pkg/types/types.go | 15 +- 17 files changed, 705 insertions(+), 23 deletions(-) create mode 100644 cmd/nodeproblemdetector/exporterplugins/default_plugin.go create mode 100644 cmd/nodeproblemdetector/exporterplugins/stackdriver_exporter_plugin.go create mode 100644 config/exporter/stackdriver-exporter.json create mode 100644 pkg/exporters/register.go create mode 100644 pkg/exporters/register_test.go create mode 100644 pkg/exporters/stackdriver/config/config.go create mode 100644 pkg/exporters/stackdriver/config/config_test.go create mode 100644 pkg/exporters/stackdriver/gce/type.go create mode 100644 pkg/exporters/stackdriver/stackdriver_exporter.go create mode 100644 pkg/exporters/stackdriver/stackdriver_exporter_test.go diff --git a/.travis.yml b/.travis.yml index e0f73de1..699ecbef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,3 +27,5 @@ script: - BUILD_TAGS="disable_system_log_monitor" make test - make clean && BUILD_TAGS="disable_system_stats_monitor" make - BUILD_TAGS="disable_system_stats_monitor" make test + - make clean && BUILD_TAGS="disable_stackdriver_exporter" make + - BUILD_TAGS="disable_stackdriver_exporter" make test diff --git a/Makefile b/Makefile index e5ff59a8..9cdf42c6 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ PKG:=k8s.io/node-problem-detector PKG_SOURCES:=$(shell find pkg cmd -name '*.go') # TARBALL is the name of release tar. Include binary version by default. -TARBALL:=node-problem-detector-$(VERSION).tar.gz +TARBALL?=node-problem-detector-$(VERSION).tar.gz # IMAGE is the image name of the node problem detector container image. IMAGE:=$(REGISTRY)/node-problem-detector:$(TAG) diff --git a/README.md b/README.md index 2fa54b55..c8e6d5c1 100644 --- a/README.md +++ b/README.md @@ -69,27 +69,44 @@ List of supported problem daemons: # Exporter An exporter is a component of node-problem-detector. It reports node problems and/or metrics to -certain back end (e.g. Kubernetes API server, or Prometheus scrape endpoint). +certain back end. Some of them can be disable at compile time using a build tag. List of supported exporters: + +| Exporter |Description | Disabling Build Tag | +|----------|:-----------|:--------------------| +| Kubernetes exporter | Kubernetes exporter reports node problems to Kubernetes API server: temporary problems get reported as Events, and permanent problems get reported as Node Conditions. | +| Prometheus exporter | Prometheus exporter reports node problems and metrics locally as Prometheus metrics | +| [Stackdriver exporter](https://github.com/kubernetes/node-problem-detector/blob/master/config/exporter/stackdriver-exporter.json) | Stackdriver exporter reports node problems and metrics to Stackdriver Monitoring API. | disable_stackdriver_exporter # Usage ## Flags * `--version`: Print current version of node-problem-detector. -* `--address`: The address to bind the node problem detector server. -* `--port`: The port to bind the node problem detector server. Use 0 to disable. +* `--hostname-override`: A customized node name used for node-problem-detector to update conditions and emit events. node-problem-detector gets node name first from `hostname-override`, then `NODE_NAME` environment variable and finally fall back to `os.Hostname`. + +#### For System Log Monitor + * `--config.system-log-monitor`: List of paths to system log monitor configuration files, comma separated, e.g. [config/kernel-monitor.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/kernel-monitor.json). Node problem detector will start a separate log monitor for each configuration. You can use different log monitors to monitor different system log. -* `--config.custom-plugin-monitor`: List of paths to custom plugin monitor config files, comma separated, e.g. - [config/custom-plugin-monitor.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/custom-plugin-monitor.json). - Node problem detector will start a separate custom plugin monitor for each configuration. You can - use different custom plugin monitors to monitor different node problems. + +#### For System Stats Monitor + * `--config.system-stats-monitor`: List of paths to system stats monitor config files, comma separated, e.g. [config/system-stats-monitor.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/system-stats-monitor.json). Node problem detector will start a separate system stats monitor for each configuration. You can use different system stats monitors to monitor different problem-related system stats. + +#### For Custom Plugin Monitor + +* `--config.custom-plugin-monitor`: List of paths to custom plugin monitor config files, comma separated, e.g. + [config/custom-plugin-monitor.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/custom-plugin-monitor.json). + Node problem detector will start a separate custom plugin monitor for each configuration. You can + use different custom plugin monitors to monitor different node problems. + +#### For Kubernetes exporter + * `--enable-k8s-exporter`: Enables reporting to Kubernetes API server, default to `true`. * `--apiserver-override`: A URI parameter used to customize how node-problem-detector connects the apiserver. This is ignored if `--enable-k8s-exporter` is `false`. The format is same as the @@ -100,10 +117,18 @@ For example, to run without auth, use the following config: http://APISERVER_IP:APISERVER_PORT?inClusterConfig=false ``` Refer [heapster docs](https://github.com/kubernetes/heapster/blob/master/docs/source-configuration.md#kubernetes) for a complete list of available options. -* `--hostname-override`: A customized node name used for node-problem-detector to update conditions and emit events. node-problem-detector gets node name first from `hostname-override`, then `NODE_NAME` environment variable and finally fall back to `os.Hostname`. +* `--address`: The address to bind the node problem detector server. +* `--port`: The port to bind the node problem detector server. Use 0 to disable. + +#### For Prometheus exporter + * `--prometheus-address`: The address to bind the Prometheus scrape endpoint, default to `127.0.0.1`. * `--prometheus-port`: The port to bind the Prometheus scrape endpoint, default to 20257. Use 0 to disable. +#### For Stackdriver exporter + +* `--exporter.stackdriver`: Path to a Stackdriver exporter config file, e.g. [config/exporter/stackdriver-exporter.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/exporter/stackdriver-exporter.json), default to empty string. Set to empty string to disable. + ### Deprecated Flags * `--system-log-monitors`: List of paths to system log monitor config files, comma separated. This option is deprecated, replaced by `--config.system-log-monitor`, and will be removed. NPD will panic if both `--system-log-monitors` and `--config.system-log-monitor` are set. diff --git a/cmd/nodeproblemdetector/exporterplugins/default_plugin.go b/cmd/nodeproblemdetector/exporterplugins/default_plugin.go new file mode 100644 index 00000000..2e348b4e --- /dev/null +++ b/cmd/nodeproblemdetector/exporterplugins/default_plugin.go @@ -0,0 +1,20 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exporterplugins + +// This file is necessary to make sure the exporterplugins package non-empty +// under any build tags. diff --git a/cmd/nodeproblemdetector/exporterplugins/stackdriver_exporter_plugin.go b/cmd/nodeproblemdetector/exporterplugins/stackdriver_exporter_plugin.go new file mode 100644 index 00000000..a580549c --- /dev/null +++ b/cmd/nodeproblemdetector/exporterplugins/stackdriver_exporter_plugin.go @@ -0,0 +1,25 @@ +// +build !disable_stackdriver_exporter + +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exporterplugins + +import ( + _ "k8s.io/node-problem-detector/pkg/exporters/stackdriver" +) + +// The stackdriver plugin takes about 6MB in the NPD binary. diff --git a/cmd/nodeproblemdetector/node_problem_detector.go b/cmd/nodeproblemdetector/node_problem_detector.go index 07185386..42fc708a 100644 --- a/cmd/nodeproblemdetector/node_problem_detector.go +++ b/cmd/nodeproblemdetector/node_problem_detector.go @@ -22,8 +22,10 @@ import ( "github.com/golang/glog" "github.com/spf13/pflag" + _ "k8s.io/node-problem-detector/cmd/nodeproblemdetector/exporterplugins" _ "k8s.io/node-problem-detector/cmd/nodeproblemdetector/problemdaemonplugins" "k8s.io/node-problem-detector/cmd/options" + "k8s.io/node-problem-detector/pkg/exporters" "k8s.io/node-problem-detector/pkg/exporters/k8sexporter" "k8s.io/node-problem-detector/pkg/exporters/prometheusexporter" "k8s.io/node-problem-detector/pkg/problemdaemon" @@ -54,21 +56,28 @@ func main() { } // Initialize exporters. - exporters := []types.Exporter{} + defaultExporters := []types.Exporter{} if ke := k8sexporter.NewExporterOrDie(npdo); ke != nil { - exporters = append(exporters, ke) + defaultExporters = append(defaultExporters, ke) glog.Info("K8s exporter started.") } if pe := prometheusexporter.NewExporterOrDie(npdo); pe != nil { - exporters = append(exporters, pe) + defaultExporters = append(defaultExporters, pe) glog.Info("Prometheus exporter started.") } - if len(exporters) == 0 { + + plugableExporters := exporters.NewExporters() + + npdExporters := []types.Exporter{} + npdExporters = append(npdExporters, defaultExporters...) + npdExporters = append(npdExporters, plugableExporters...) + + if len(npdExporters) == 0 { glog.Fatalf("No exporter is successfully setup") } // Initialize NPD core. - p := problemdetector.NewProblemDetector(problemDaemons, exporters) + p := problemdetector.NewProblemDetector(problemDaemons, npdExporters) if err := p.Run(); err != nil { glog.Fatalf("Problem detector failed with error: %v", err) } diff --git a/cmd/options/options.go b/cmd/options/options.go index ab6ccdbc..acc8327d 100644 --- a/cmd/options/options.go +++ b/cmd/options/options.go @@ -26,6 +26,7 @@ import ( "github.com/spf13/pflag" + "k8s.io/node-problem-detector/pkg/exporters" "k8s.io/node-problem-detector/pkg/problemdaemon" "k8s.io/node-problem-detector/pkg/types" ) @@ -86,6 +87,7 @@ type NodeProblemDetectorOptions struct { func NewNodeProblemDetectorOptions() *NodeProblemDetectorOptions { npdo := &NodeProblemDetectorOptions{MonitorConfigPaths: types.ProblemDaemonConfigPathMap{}} + for _, problemDaemonName := range problemdaemon.GetProblemDaemonNames() { npdo.MonitorConfigPaths[problemDaemonName] = &[]string{} } @@ -118,6 +120,10 @@ func (npdo *NodeProblemDetectorOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&npdo.PrometheusServerAddress, "prometheus-address", "127.0.0.1", "The address to bind the Prometheus scrape endpoint.") + for _, exporterName := range exporters.GetExporterNames() { + exporterHandler := exporters.GetExporterHandlerOrDie(exporterName) + exporterHandler.Options.SetFlags(fs) + } for _, problemDaemonName := range problemdaemon.GetProblemDaemonNames() { fs.StringSliceVar( npdo.MonitorConfigPaths[problemDaemonName], diff --git a/config/exporter/stackdriver-exporter.json b/config/exporter/stackdriver-exporter.json new file mode 100644 index 00000000..a3ef0dda --- /dev/null +++ b/config/exporter/stackdriver-exporter.json @@ -0,0 +1,8 @@ +{ + "apiEndpoint": "monitoring.googleapis.com:443", + "exportPeriod": "60s", + "metadataFetchTimeout": "600s", + "metadataFetchInterval": "10s", + "panicOnMetadataFetchFailure": false, + "customMetricPrefix": "" +} diff --git a/config/systemd/node-problem-detector-metric-only.service b/config/systemd/node-problem-detector-metric-only.service index 142fee17..2503bb13 100644 --- a/config/systemd/node-problem-detector-metric-only.service +++ b/config/systemd/node-problem-detector-metric-only.service @@ -1,12 +1,13 @@ [Unit] Description=Node problem detector -Wants=local-fs.target -After=local-fs.target +Wants=network-online.target +After=network-online.target [Service] Restart=always RestartSec=10 ExecStart=/home/kubernetes/bin/node-problem-detector --v=2 --logtostderr --enable-k8s-exporter=false \ + --exporter.stackdriver=/home/kubernetes/node-problem-detector/config/exporter/stackdriver-exporter.json \ --config.system-log-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor.json,/home/kubernetes/node-problem-detector/config/docker-monitor.json,/home/kubernetes/node-problem-detector/config/systemd-monitor.json \ --config.custom-plugin-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor-counter.json,/home/kubernetes/node-problem-detector/config/systemd-monitor-counter.json \ --config.system-stats-monitor=/home/kubernetes/node-problem-detector/config/system-stats-monitor.json diff --git a/pkg/exporters/register.go b/pkg/exporters/register.go new file mode 100644 index 00000000..82839dad --- /dev/null +++ b/pkg/exporters/register.go @@ -0,0 +1,63 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exporters + +import ( + "fmt" + + "k8s.io/node-problem-detector/pkg/types" +) + +var ( + handlers = make(map[types.ExporterType]types.ExporterHandler) +) + +// Register registers a exporter factory method, which will be used to create the exporter. +func Register(exporterType types.ExporterType, handler types.ExporterHandler) { + handlers[exporterType] = handler +} + +// GetExporterNames retrieves all available exporter types. +func GetExporterNames() []types.ExporterType { + exporterTypes := []types.ExporterType{} + for exporterType := range handlers { + exporterTypes = append(exporterTypes, exporterType) + } + return exporterTypes +} + +// GetExporterHandlerOrDie retrieves the ExporterHandler for a specific type of exporter, panic if error occurs.. +func GetExporterHandlerOrDie(exporterType types.ExporterType) types.ExporterHandler { + handler, ok := handlers[exporterType] + if !ok { + panic(fmt.Sprintf("Exporter handler for %v does not exist", exporterType)) + } + return handler +} + +// NewExporters creates all exporters based on the configurations initialized. +func NewExporters() []types.Exporter { + exporters := []types.Exporter{} + for _, handler := range handlers { + exporter := handler.CreateExporterOrDie(handler.Options) + if exporter == nil { + continue + } + exporters = append(exporters, exporter) + } + return exporters +} diff --git a/pkg/exporters/register_test.go b/pkg/exporters/register_test.go new file mode 100644 index 00000000..8a688425 --- /dev/null +++ b/pkg/exporters/register_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exporters + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/node-problem-detector/pkg/types" +) + +func TestRegistration(t *testing.T) { + fooExporterFactory := func(types.CommandLineOptions) types.Exporter { + return nil + } + fooExporterHandler := types.ExporterHandler{ + CreateExporterOrDie: fooExporterFactory, + Options: nil, + } + + barExporterFactory := func(types.CommandLineOptions) types.Exporter { + return nil + } + barExporterHandler := types.ExporterHandler{ + CreateExporterOrDie: barExporterFactory, + Options: nil, + } + + Register("foo", fooExporterHandler) + Register("bar", barExporterHandler) + + expectedExporterNames := []types.ExporterType{"foo", "bar"} + exporterNames := GetExporterNames() + assert.ElementsMatch(t, expectedExporterNames, exporterNames) + + handlers = make(map[types.ExporterType]types.ExporterHandler) +} + +func TestGetExporterHandlerOrDie(t *testing.T) { + fooExporterFactory := func(types.CommandLineOptions) types.Exporter { + return nil + } + fooExporterHandler := types.ExporterHandler{ + CreateExporterOrDie: fooExporterFactory, + Options: nil, + } + + Register("foo", fooExporterHandler) + + assert.NotPanics(t, func() { GetExporterHandlerOrDie("foo") }) + assert.Panics(t, func() { GetExporterHandlerOrDie("bar") }) + + handlers = make(map[types.ExporterType]types.ExporterHandler) +} diff --git a/pkg/exporters/stackdriver/config/config.go b/pkg/exporters/stackdriver/config/config.go new file mode 100644 index 00000000..ab03cd98 --- /dev/null +++ b/pkg/exporters/stackdriver/config/config.go @@ -0,0 +1,56 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "time" + + "k8s.io/node-problem-detector/pkg/exporters/stackdriver/gce" +) + +var ( + defaultExportPeriod = (60 * time.Second).String() + defaultEndpoint = "monitoring.googleapis.com:443" + defaultMetadataFetchTimeout = (600 * time.Second).String() + defaultMetadataFetchInterval = (10 * time.Second).String() +) + +type StackdriverExporterConfig struct { + ExportPeriod string `json:"exportPeriod"` + APIEndpoint string `json:"apiEndpoint"` + GCEMetadata gce.Metadata `json:"gceMetadata"` + MetadataFetchTimeout string `json:"metadataFetchTimeout"` + MetadataFetchInterval string `json:"metadataFetchInterval"` + PanicOnMetadataFetchFailure bool `json:"panicOnMetadataFetchFailure"` + CustomMetricPrefix string `json:"customMetricPrefix"` +} + +// ApplyConfiguration applies default configurations. +func (sec *StackdriverExporterConfig) ApplyConfiguration() { + if sec.ExportPeriod == "" { + sec.ExportPeriod = defaultExportPeriod + } + if sec.MetadataFetchTimeout == "" { + sec.MetadataFetchTimeout = defaultMetadataFetchTimeout + } + if sec.MetadataFetchInterval == "" { + sec.MetadataFetchInterval = defaultMetadataFetchInterval + } + if sec.APIEndpoint == "" { + sec.APIEndpoint = defaultEndpoint + } +} diff --git a/pkg/exporters/stackdriver/config/config_test.go b/pkg/exporters/stackdriver/config/config_test.go new file mode 100644 index 00000000..87c7b4de --- /dev/null +++ b/pkg/exporters/stackdriver/config/config_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "reflect" + "testing" + + "k8s.io/node-problem-detector/pkg/exporters/stackdriver/gce" +) + +func TestApplyConfiguration(t *testing.T) { + testCases := []struct { + name string + orignalConfig StackdriverExporterConfig + wantedConfig StackdriverExporterConfig + }{ + { + name: "normal", + orignalConfig: StackdriverExporterConfig{ + ExportPeriod: "60s", + MetadataFetchTimeout: "600s", + MetadataFetchInterval: "10s", + APIEndpoint: "monitoring.googleapis.com:443", + GCEMetadata: gce.Metadata{ + ProjectID: "some-gcp-project", + Zone: "us-central1-a", + InstanceID: "56781234", + InstanceName: "some-gce-instance", + }, + }, + wantedConfig: StackdriverExporterConfig{ + ExportPeriod: "60s", + MetadataFetchTimeout: "600s", + MetadataFetchInterval: "10s", + APIEndpoint: defaultEndpoint, + GCEMetadata: gce.Metadata{ + ProjectID: "some-gcp-project", + Zone: "us-central1-a", + InstanceID: "56781234", + InstanceName: "some-gce-instance", + }, + }, + }, + { + name: "staging API endpoint", + orignalConfig: StackdriverExporterConfig{ + ExportPeriod: "60s", + MetadataFetchTimeout: "600s", + MetadataFetchInterval: "10s", + APIEndpoint: "staging-monitoring.sandbox.googleapis.com:443", + GCEMetadata: gce.Metadata{ + ProjectID: "some-gcp-project", + Zone: "us-central1-a", + InstanceID: "56781234", + InstanceName: "some-gce-instance", + }, + }, + wantedConfig: StackdriverExporterConfig{ + ExportPeriod: "60s", + MetadataFetchTimeout: "600s", + MetadataFetchInterval: "10s", + APIEndpoint: "staging-monitoring.sandbox.googleapis.com:443", + GCEMetadata: gce.Metadata{ + ProjectID: "some-gcp-project", + Zone: "us-central1-a", + InstanceID: "56781234", + InstanceName: "some-gce-instance", + }, + }, + }, + { + name: "empty", + orignalConfig: StackdriverExporterConfig{}, + wantedConfig: StackdriverExporterConfig{ + ExportPeriod: "1m0s", + MetadataFetchTimeout: "10m0s", + MetadataFetchInterval: "10s", + APIEndpoint: "monitoring.googleapis.com:443", + GCEMetadata: gce.Metadata{}, + }, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + test.orignalConfig.ApplyConfiguration() + if !reflect.DeepEqual(test.orignalConfig, test.wantedConfig) { + t.Errorf("Wanted: %+v. \nGot: %+v", test.wantedConfig, test.orignalConfig) + } + }) + } +} diff --git a/pkg/exporters/stackdriver/gce/type.go b/pkg/exporters/stackdriver/gce/type.go new file mode 100644 index 00000000..d486286c --- /dev/null +++ b/pkg/exporters/stackdriver/gce/type.go @@ -0,0 +1,67 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "cloud.google.com/go/compute/metadata" + "github.com/golang/glog" +) + +type Metadata struct { + ProjectID string `json:"projectID"` + Zone string `json:"zone"` + InstanceID string `json:"instanceID"` + InstanceName string `json:"instanceName"` +} + +func (md *Metadata) HasMissingField() bool { + if md.ProjectID == "" || md.Zone == "" || md.InstanceID == "" || md.InstanceName == "" { + return true + } + return false +} + +func (md *Metadata) PopulateFromGCE() error { + var err error + glog.Info("Fetching GCE metadata from metadata server") + if md.ProjectID == "" { + md.ProjectID, err = metadata.ProjectID() + if err != nil { + return err + } + } + if md.Zone == "" { + md.Zone, err = metadata.Zone() + if err != nil { + return err + } + } + if md.InstanceID == "" { + md.InstanceID, err = metadata.InstanceID() + if err != nil { + return err + } + } + if md.InstanceName == "" { + md.InstanceName, err = metadata.InstanceName() + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/exporters/stackdriver/stackdriver_exporter.go b/pkg/exporters/stackdriver/stackdriver_exporter.go new file mode 100644 index 00000000..cae4be93 --- /dev/null +++ b/pkg/exporters/stackdriver/stackdriver_exporter.go @@ -0,0 +1,190 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stackdriverexporter + +import ( + "encoding/json" + "io/ioutil" + "path/filepath" + "reflect" + "time" + + "contrib.go.opencensus.io/exporter/stackdriver" + monitoredres "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" + "github.com/golang/glog" + "github.com/spf13/pflag" + "go.opencensus.io/stats/view" + "google.golang.org/api/option" + + "github.com/avast/retry-go" + "k8s.io/node-problem-detector/pkg/exporters" + seconfig "k8s.io/node-problem-detector/pkg/exporters/stackdriver/config" + "k8s.io/node-problem-detector/pkg/types" + "k8s.io/node-problem-detector/pkg/util/metrics" +) + +func init() { + clo := commandLineOptions{} + exporters.Register(exporterName, types.ExporterHandler{ + CreateExporterOrDie: NewExporterOrDie, + Options: &clo}) +} + +const exporterName = "stackdriver" + +var NPDMetricToSDMetric = map[metrics.MetricID]string{ + metrics.HostUptimeID: "compute.googleapis.com/guest/system/uptime", + metrics.ProblemCounterID: "compute.googleapis.com/guest/system/problem_count", + metrics.DiskAvgQueueLenID: "compute.googleapis.com/guest/disk/queue_length", + metrics.DiskIOTimeID: "compute.googleapis.com/guest/disk/io_time", + metrics.DiskWeightedIOID: "compute.googleapis.com/guest/disk/weighted_io_time", +} + +func getMetricTypeConversionFunction(customMetricPrefix string) func(*view.View) string { + + return func(view *view.View) string { + viewName := view.Measure.Name() + + fallbackMetricType := "" + if customMetricPrefix != "" { + // Example fallbackMetricType: custom.googleapis.com/npd/host/uptime + fallbackMetricType = filepath.Join(customMetricPrefix, viewName) + } + + metricID, ok := metrics.MetricMap.ViewNameToMetricID(viewName) + if !ok { + return fallbackMetricType + } + stackdriverMetricType, ok := NPDMetricToSDMetric[metricID] + if !ok { + return fallbackMetricType + } + return stackdriverMetricType + } +} + +type stackdriverExporter struct { + config seconfig.StackdriverExporterConfig +} + +func (se *stackdriverExporter) setupOpenCensusViewExporterOrDie() { + clientOption := option.WithEndpoint(se.config.APIEndpoint) + + var globalLabels stackdriver.Labels + globalLabels.Set("instance_name", se.config.GCEMetadata.InstanceName, "The name of the VM instance") + + viewExporter, err := stackdriver.NewExporter(stackdriver.Options{ + ProjectID: se.config.GCEMetadata.ProjectID, + MonitoringClientOptions: []option.ClientOption{clientOption}, + MonitoredResource: &monitoredres.GCEInstance{ + ProjectID: se.config.GCEMetadata.ProjectID, + InstanceID: se.config.GCEMetadata.InstanceID, + Zone: se.config.GCEMetadata.Zone, + }, + GetMetricType: getMetricTypeConversionFunction(se.config.CustomMetricPrefix), + DefaultMonitoringLabels: &globalLabels, + }) + if err != nil { + glog.Fatalf("Failed to create Stackdriver OpenCensus view exporter: %v", err) + } + + exportPeriod, err := time.ParseDuration(se.config.ExportPeriod) + if err != nil { + glog.Fatalf("Failed to parse ExportPeriod %q: %v", se.config.ExportPeriod, err) + } + + view.SetReportingPeriod(exportPeriod) + view.RegisterExporter(viewExporter) +} + +func (se *stackdriverExporter) populateMetadataOrDie() { + if !se.config.GCEMetadata.HasMissingField() { + glog.Infof("Using GCE metadata specified in the config file: %+v", se.config.GCEMetadata) + return + } + + metadataFetchTimeout, err := time.ParseDuration(se.config.MetadataFetchTimeout) + if err != nil { + glog.Fatalf("Failed to parse MetadataFetchTimeout %q: %v", se.config.MetadataFetchTimeout, err) + } + + metadataFetchInterval, err := time.ParseDuration(se.config.MetadataFetchInterval) + if err != nil { + glog.Fatalf("Failed to parse MetadataFetchInterval %q: %v", se.config.MetadataFetchInterval, err) + } + + glog.Infof("Populating GCE metadata by querying GCE metadata server.") + err = retry.Do(se.config.GCEMetadata.PopulateFromGCE, + retry.Delay(metadataFetchInterval), + retry.Attempts(uint(metadataFetchTimeout/metadataFetchInterval)), + retry.DelayType(retry.FixedDelay)) + if err == nil { + glog.Infof("Using GCE metadata: %+v", se.config.GCEMetadata) + return + } + if se.config.PanicOnMetadataFetchFailure { + glog.Fatalf("Failed to populate GCE metadata: %v", err) + } else { + glog.Errorf("Failed to populate GCE metadata: %v", err) + } +} + +// ExportProblems does nothing. +// Stackdriver exporter only exports metrics. +func (se *stackdriverExporter) ExportProblems(status *types.Status) { + return +} + +type commandLineOptions struct { + configPath string +} + +func (clo *commandLineOptions) SetFlags(fs *pflag.FlagSet) { + fs.StringVar(&clo.configPath, "exporter.stackdriver", "", + "Configuration for Stackdriver exporter. Set to config file path.") +} + +// NewExporterOrDie creates an exporter to export metrics to Stackdriver, panics if error occurs. +func NewExporterOrDie(clo types.CommandLineOptions) types.Exporter { + options, ok := clo.(*commandLineOptions) + if !ok { + glog.Fatalf("Wrong type for the command line options of Stackdriver Exporter: %s.", reflect.TypeOf(clo)) + } + if options.configPath == "" { + return nil + } + + se := stackdriverExporter{} + + // Apply configurations. + f, err := ioutil.ReadFile(options.configPath) + if err != nil { + glog.Fatalf("Failed to read configuration file %q: %v", options.configPath, err) + } + err = json.Unmarshal(f, &se.config) + if err != nil { + glog.Fatalf("Failed to unmarshal configuration file %q: %v", options.configPath, err) + } + se.config.ApplyConfiguration() + + glog.Infof("Starting Stackdriver exporter %s", options.configPath) + + se.populateMetadataOrDie() + se.setupOpenCensusViewExporterOrDie() + + return &se +} diff --git a/pkg/exporters/stackdriver/stackdriver_exporter_test.go b/pkg/exporters/stackdriver/stackdriver_exporter_test.go new file mode 100644 index 00000000..f5f39481 --- /dev/null +++ b/pkg/exporters/stackdriver/stackdriver_exporter_test.go @@ -0,0 +1,33 @@ +// +build !disable_stackdriver_exporter + +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stackdriverexporter + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/node-problem-detector/pkg/exporters" +) + +func TestRegistration(t *testing.T) { + assert.NotPanics(t, + func() { exporters.GetExporterHandlerOrDie(exporterName) }, + "Stackdriver exporter failed to register itself as an exporter.") +} diff --git a/pkg/types/types.go b/pkg/types/types.go index ee422a36..1a36804d 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -18,6 +18,8 @@ package types import ( "time" + + "github.com/spf13/pflag" ) // The following types are used internally in problem detector. In the future this could be the @@ -135,15 +137,14 @@ type ProblemDaemonHandler struct { // ExporterType is the type of the exporter. type ExporterType string -// ExporterConfigPathMap represents configurations on all types of exporters: -// 1) Each key represents a type of exporter. -// 2) Each value represents the config file path for the exporter. -type ExporterConfigPathMap map[ExporterType]*string - // ExporterHandler represents the initialization handler for a type of exporter. type ExporterHandler struct { // CreateExporterOrDie initializes an exporter, panic if error occurs. - CreateExporterOrDie func(string) Exporter + CreateExporterOrDie func(CommandLineOptions) Exporter // CmdOptionDescription explains how to configure the exporter from command line arguments. - CmdOptionDescription string + Options CommandLineOptions +} + +type CommandLineOptions interface { + SetFlags(*pflag.FlagSet) }