Support waiting for kube-apiserver to be ready with timout during NPD startup

cmd/node_problem_detector.go

@@ -26,6 +26,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/spf13/pflag"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/node-problem-detector/cmd/options"
 	"k8s.io/node-problem-detector/pkg/custompluginmonitor"
 	"k8s.io/node-problem-detector/pkg/problemclient"
@@ -95,7 +96,25 @@ func main() {
 		startHTTPServer(p, npdo)
 	}
 
+	// This function may be blocked (until a timeout occurs) before
+	// kube-apiserver becomes ready.
+	glog.Infof("Waiting for kube-apiserver to be ready (timeout %v)...", npdo.APIServerWaitTimeout)
+	if err := waitForAPIServerReadyWithTimeout(c, npdo); err != nil {
+		glog.Warningf("kube-apiserver did not become ready: timed out on waiting for kube-apiserver to return the node object: %v", err)
+	}
+
 	if err := p.Run(); err != nil {
 		glog.Fatalf("Problem detector failed with error: %v", err)
 	}
 }
+
+func waitForAPIServerReadyWithTimeout(c problemclient.Client, npdo *options.NodeProblemDetectorOptions) error {
+	return wait.PollImmediate(npdo.APIServerWaitInterval, npdo.APIServerWaitTimeout, func() (done bool, err error) {
+		// If NPD can get the node object from kube-apiserver, the server is
+		// ready and the RBAC permission is set correctly.
+		if _, err := c.GetNode(); err == nil {
+			return true, nil
+		}
+		return false, nil
+	})
+}

cmd/options/options.go

@@ -20,6 +20,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"time"
 
 	"net/url"
 
@@ -46,6 +47,12 @@ type NodeProblemDetectorOptions struct {
 	ServerPort int
 	// ServerAddress is the address to bind the node problem detector server.
 	ServerAddress string
+	// APIServerWaitTimeout is the timeout on waiting for kube-apiserver to be
+	// ready.
+	APIServerWaitTimeout time.Duration
+	// APIServerWaitInterval is the interval between the checks on the
+	// readiness of kube-apiserver.
+	APIServerWaitInterval time.Duration
 
 	// application options
 
@@ -65,6 +72,8 @@ func (npdo *NodeProblemDetectorOptions) AddFlags(fs *pflag.FlagSet) {
 		[]string{}, "List of paths to custom plugin monitor config files, comma separated.")
 	fs.StringVar(&npdo.ApiServerOverride, "apiserver-override",
 		"", "Custom URI used to connect to Kubernetes ApiServer")
+	fs.DurationVar(&npdo.APIServerWaitTimeout, "apiserver-wait-timeout", time.Duration(5)*time.Minute, "The timeout on waiting for kube-apiserver to be ready. This is ignored if --enable-k8s-exporter is false.")
+	fs.DurationVar(&npdo.APIServerWaitInterval, "apiserver-wait-interval", time.Duration(5)*time.Second, "The interval between the checks on the readiness of kube-apiserver. This is ignored if --enable-k8s-exporter is false.")
 	fs.BoolVar(&npdo.PrintVersion, "version", false, "Print version information and quit")
 	fs.StringVar(&npdo.HostnameOverride, "hostname-override",
 		"", "Custom node name used to override hostname")