diff --git a/README.md b/README.md index daeef20..4b3bb01 100644 --- a/README.md +++ b/README.md @@ -83,9 +83,13 @@ The following arguments can be passed to kured via the daemonset pod template: Flags: --alert-filter-regexp regexp.Regexp alert names to ignore when checking for active alerts --blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots + --drain-grace-period int time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1) + --skip-wait-for-delete-timeout int when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0) --ds-name string name of daemonset on which to place lock (default "kured") --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") --end-time string schedule reboot only before this time of day (default "23:59:59") + --force-reboot bool force a reboot even if the drain is still running (default: false) + --drain-timeout duration timeout after which the drain is aborted (default: 0, infinite time) -h, --help help for kured --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") --lock-ttl duration expire lock annotation after this duration (default: 0, disabled) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index fe7a784..6f6cfeb 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -38,25 +38,29 @@ var ( version = "unreleased" // Command line flags - period time.Duration - dsNamespace string - dsName string - lockAnnotation string - lockTTL time.Duration - lockReleaseDelay time.Duration - prometheusURL string - preferNoScheduleTaintName string - alertFilter *regexp.Regexp - rebootSentinelFile string - rebootSentinelCommand string - notifyURL string - slackHookURL string - slackUsername string - slackChannel string - messageTemplateDrain string - messageTemplateReboot string - podSelectors []string - rebootCommand string + forceReboot bool + drainTimeout time.Duration + period time.Duration + drainGracePeriod int + skipWaitForDeleteTimeoutSeconds int + dsNamespace string + dsName string + lockAnnotation string + lockTTL time.Duration + lockReleaseDelay time.Duration + prometheusURL string + preferNoScheduleTaintName string + alertFilter *regexp.Regexp + rebootSentinelFile string + rebootSentinelCommand string + notifyURL string + slackHookURL string + slackUsername string + slackChannel string + messageTemplateDrain string + messageTemplateReboot string + podSelectors []string + rebootCommand string rebootDays []string rebootStart string @@ -92,6 +96,14 @@ func main() { PreRun: flagCheck, Run: root} + rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false, + "force a reboot even if the drain is still running (default: false)") + rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1, + "time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1)") + rootCmd.PersistentFlags().IntVar(&skipWaitForDeleteTimeoutSeconds, "skip-wait-for-delete-timeout", 0, + "when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0)") + rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 0, + "timeout after which the drain is aborted (default: 0, infinite time)") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, "sentinel check period") rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system", @@ -310,6 +322,7 @@ func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}, TTL time.D return true } } + func throttle(releaseDelay time.Duration) { if releaseDelay > 0 { log.Infof("Delaying lock release by %v", releaseDelay) @@ -341,20 +354,32 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { } drainer := &kubectldrain.Helper{ - Client: client, - GracePeriodSeconds: -1, - Force: true, - DeleteEmptyDirData: true, - IgnoreAllDaemonSets: true, - ErrOut: os.Stderr, - Out: os.Stdout, + Client: client, + Ctx: context.Background(), + GracePeriodSeconds: drainGracePeriod, + SkipWaitForDeleteTimeoutSeconds: skipWaitForDeleteTimeoutSeconds, + Force: true, + DeleteEmptyDirData: true, + IgnoreAllDaemonSets: true, + ErrOut: os.Stderr, + Out: os.Stdout, + Timeout: drainTimeout, } + if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { - log.Fatalf("Error cordonning %s: %v", nodename, err) + if !forceReboot { + log.Fatalf("Error cordonning %s: %v", nodename, err) + } + log.Errorf("Error cordonning %s: %v, continuing with reboot anyway", nodename, err) + return } if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil { - log.Fatalf("Error draining %s: %v", nodename, err) + if !forceReboot { + log.Fatalf("Error draining %s: %v", nodename, err) + } + log.Errorf("Error draining %s: %v, continuing with reboot anyway", nodename, err) + return } } diff --git a/pkg/daemonsetlock/daemonsetlock.go b/pkg/daemonsetlock/daemonsetlock.go index eb038be..631d0df 100644 --- a/pkg/daemonsetlock/daemonsetlock.go +++ b/pkg/daemonsetlock/daemonsetlock.go @@ -6,11 +6,18 @@ import ( "fmt" "time" + v1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" ) +const ( + k8sAPICallRetrySleep = 5 * time.Second // How much time to wait in between retrying a k8s API call + k8sAPICallRetryTimeout = 5 * time.Minute // How long to wait until we determine that the k8s API is definitively unavailable +) + // DaemonSetLock holds all necessary information to do actions // on the kured ds which holds lock info through annotations. type DaemonSetLock struct { @@ -34,11 +41,11 @@ func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation strin } // Acquire attempts to annotate the kured daemonset with lock info from instantiated DaemonSetLock using client-go -func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acquired bool, owner string, err error) { +func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (bool, string, error) { for { - ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{}) + ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout) if err != nil { - return false, "", err + return false, "", fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err) } valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] @@ -78,10 +85,10 @@ func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acqu } // Test attempts to check the kured daemonset lock status (existence, expiry) from instantiated DaemonSetLock using client-go -func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) { - ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{}) +func (dsl *DaemonSetLock) Test(metadata interface{}) (bool, error) { + ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout) if err != nil { - return false, err + return false, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err) } valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] @@ -102,9 +109,9 @@ func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) { // Release attempts to remove the lock data from the kured ds annotations using client-go func (dsl *DaemonSetLock) Release() error { for { - ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{}) + ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout) if err != nil { - return err + return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err) } valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] @@ -137,6 +144,24 @@ func (dsl *DaemonSetLock) Release() error { } } +// GetDaemonSet returns the named DaemonSet resource from the DaemonSetLock's configured client +func (dsl *DaemonSetLock) GetDaemonSet(sleep, timeout time.Duration) (*v1.DaemonSet, error) { + var ds *v1.DaemonSet + var lastError error + err := wait.PollImmediate(sleep, timeout, func() (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + if ds, lastError = dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(ctx, dsl.name, metav1.GetOptions{}); lastError != nil { + return false, nil + } + return true, nil + }) + if err != nil { + return nil, fmt.Errorf("Timed out trying to get daemonset %s in namespace %s: %v", dsl.name, dsl.namespace, lastError) + } + return ds, nil +} + func ttlExpired(created time.Time, ttl time.Duration) bool { if ttl > 0 && time.Since(created) >= ttl { return true