diff --git a/README.md b/README.md index 78c5764..b3abe33 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ * [Installation](#installation) * [Configuration](#configuration) * [Reboot Sentinel File & Period](#reboot-sentinel-file-&-period) + * [Setting a schedule](#setting-a-schedule) * [Blocking Reboots via Alerts](#blocking-reboots-via-alerts) * [Blocking Reboots via Pods](#blocking-reboots-via-pods) * [Prometheus Metrics](#prometheus-metrics) @@ -74,14 +75,18 @@ Flags: --blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots --ds-name string name of daemonset on which to place lock (default "kured") --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") + --end-time string only reboot before this time of day (default "23:59") -h, --help help for kured --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") --period duration reboot check period (default 1h0m0s) --prometheus-url string Prometheus instance to probe for active alerts + --reboot-days strings only reboot on these days (default [su,mo,tu,we,th,fr,sa]) --reboot-sentinel string path to file whose existence signals need to reboot (default "/var/run/reboot-required") --slack-channel string slack channel for reboot notfications --slack-hook-url string slack hook URL for reboot notfications --slack-username string slack username for reboot notfications (default "kured") + --start-time string only reboot after this time of day (default "0:00") + --time-zone string use this timezone to calculate allowed reboot time (default "UTC") ``` ### Reboot Sentinel File & Period @@ -92,6 +97,29 @@ values with `--reboot-sentinel` and `--period`. Each replica of the daemon uses a random offset derived from the period on startup so that nodes don't all contend for the lock simultaneously. +### Setting a schedule + +By default, kured will reboot any time it detects the sentinel, but this +may cause reboots during odd hours. While service disruption does not +normally occur, anything is possible and operators may want to restrict +reboots to predictable schedules. Use `--reboot-days`, `--start-time`, +`--end-time`, and `--time-zone` to set a schedule. For example, business +hours on the west coast USA can be specified with: + +``` + --reboot-days mon,tue,wed,thu,fri + --start-time 9am + --end-time 5pm + --time-zone America/Los_Angeles +``` + +Times can be formatted in numerous ways, including `5pm`, `5:00pm` `17:00`, +and `17`. `--time-zone` represents a Go `time.Location`, and can be `UTC`, +`Local`, or any entry in the standard Linux tz database. + +Note that when using smaller time windows, you should consider shortening +the sentinel check period (`--period`). + ### Blocking Reboots via Alerts You may find it desirable to block automatic node reboots when there diff --git a/cmd/kured/Dockerfile b/cmd/kured/Dockerfile index 272dde7..adfdd4f 100644 --- a/cmd/kured/Dockerfile +++ b/cmd/kured/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.8 -RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/* +RUN apk update && apk add ca-certificates tzdata && rm -rf /var/cache/apk/* # NB: you may need to update RBAC permissions when upgrading kubectl - see kured-rbac.yaml for details ADD https://storage.googleapis.com/kubernetes-release/release/v1.14.1/bin/linux/amd64/kubectl /usr/bin/kubectl RUN chmod 0755 /usr/bin/kubectl diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 1d6ee0d..387e609 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -21,6 +21,7 @@ import ( "github.com/weaveworks/kured/pkg/daemonsetlock" "github.com/weaveworks/kured/pkg/delaytick" "github.com/weaveworks/kured/pkg/notifications/slack" + "github.com/weaveworks/kured/pkg/timewindow" ) var ( @@ -39,6 +40,11 @@ var ( slackChannel string podSelectors []string + rebootDays []string + rebootStart string + rebootEnd string + timezone string + // Metrics rebootRequiredGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Subsystem: "kured", @@ -82,6 +88,15 @@ func main() { rootCmd.PersistentFlags().StringArrayVar(&podSelectors, "blocking-pod-selector", nil, "label selector identifying pods whose presence should prevent reboots") + rootCmd.PersistentFlags().StringSliceVar(&rebootDays, "reboot-days", timewindow.EveryDay, + "schedule reboot on these days") + rootCmd.PersistentFlags().StringVar(&rebootStart, "start-time", "0:00", + "schedule reboot only after this time of day") + rootCmd.PersistentFlags().StringVar(&rebootEnd, "end-time", "23:59:59", + "schedule reboot only before this time of day") + rootCmd.PersistentFlags().StringVar(&timezone, "time-zone", "UTC", + "use this timezone for schedule inputs") + if err := rootCmd.Execute(); err != nil { log.Fatal(err) } @@ -268,7 +283,7 @@ type nodeMeta struct { Unschedulable bool `json:"unschedulable"` } -func rebootAsRequired(nodeID string) { +func rebootAsRequired(nodeID string, window *timewindow.TimeWindow) { config, err := rest.InClusterConfig() if err != nil { log.Fatal(err) @@ -292,7 +307,7 @@ func rebootAsRequired(nodeID string) { source := rand.NewSource(time.Now().UnixNano()) tick := delaytick.New(source, period) for _ = range tick { - if rebootRequired() && !rebootBlocked(client, nodeID) { + if window.Contains(time.Now()) && rebootRequired() && !rebootBlocked(client, nodeID) { node, err := client.CoreV1().Nodes().Get(nodeID, metav1.GetOptions{}) if err != nil { log.Fatal(err) @@ -321,12 +336,18 @@ func root(cmd *cobra.Command, args []string) { log.Fatal("KURED_NODE_ID environment variable required") } + window, err := timewindow.New(rebootDays, rebootStart, rebootEnd, timezone) + if err != nil { + log.Fatalf("Failed to build time window: %v", err) + } + log.Infof("Node ID: %s", nodeID) log.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation) log.Infof("Reboot Sentinel: %s every %v", rebootSentinel, period) log.Infof("Blocking Pod Selectors: %v", podSelectors) + log.Infof("Reboot on: %v", window) - go rebootAsRequired(nodeID) + go rebootAsRequired(nodeID, window) go maintainRebootRequiredMetric(nodeID) http.Handle("/metrics", promhttp.Handler()) diff --git a/kured-ds.yaml b/kured-ds.yaml index 42879f3..d8e160e 100644 --- a/kured-ds.yaml +++ b/kured-ds.yaml @@ -51,10 +51,14 @@ spec: # - --blocking-pod-selector=... # - --ds-name=kured # - --ds-namespace=kube-system +# - --end-time=23:59:59 # - --lock-annotation=weave.works/kured-node-lock # - --period=1h # - --prometheus-url=http://prometheus.monitoring.svc.cluster.local +# - --reboot-days=sun,mon,tue,wed,thu,fri,sat # - --reboot-sentinel=/var/run/reboot-required # - --slack-hook-url=https://hooks.slack.com/... # - --slack-username=prod # - --slack-channel=alerting +# - --start-time=0:00 +# - --time-zone=UTC diff --git a/pkg/timewindow/days.go b/pkg/timewindow/days.go new file mode 100644 index 0000000..2635c84 --- /dev/null +++ b/pkg/timewindow/days.go @@ -0,0 +1,91 @@ +package timewindow + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +var EveryDay = []string{"su", "mo", "tu", "we", "th", "fr", "sa"} + +// dayStrings maps day strings to time.Weekdays +var dayStrings = map[string]time.Weekday{ + "su": time.Sunday, + "sun": time.Sunday, + "sunday": time.Sunday, + "mo": time.Monday, + "mon": time.Monday, + "monday": time.Monday, + "tu": time.Tuesday, + "tue": time.Tuesday, + "tuesday": time.Tuesday, + "we": time.Wednesday, + "wed": time.Wednesday, + "wednesday": time.Wednesday, + "th": time.Thursday, + "thu": time.Thursday, + "thursday": time.Thursday, + "fr": time.Friday, + "fri": time.Friday, + "friday": time.Friday, + "sa": time.Saturday, + "sat": time.Saturday, + "saturday": time.Saturday, +} + +type weekdays uint32 + +// parseWeekdays creates a set of weekdays from a string slice +func parseWeekdays(days []string) (weekdays, error) { + var result uint32 + for _, day := range days { + if len(day) == 0 { + continue + } + + weekday, err := parseWeekday(day) + if err != nil { + return weekdays(0), err + } + + result |= 1 << uint32(weekday) + } + + return weekdays(result), nil +} + +// Contains returns true if the specified weekday is a member of this set. +func (w weekdays) Contains(day time.Weekday) bool { + return uint32(w)&(1<= 0 && n < 7 { + return time.Weekday(n), nil + } else { + return time.Sunday, fmt.Errorf("Invalid weekday, number out of range: %s", day) + } + } + + if weekday, ok := dayStrings[strings.ToLower(day)]; ok { + return weekday, nil + } else { + return time.Sunday, fmt.Errorf("Invalid weekday: %s", day) + } +} diff --git a/pkg/timewindow/days_test.go b/pkg/timewindow/days_test.go new file mode 100644 index 0000000..8f5f00a --- /dev/null +++ b/pkg/timewindow/days_test.go @@ -0,0 +1,46 @@ +package timewindow + +import ( + "strings" + "testing" +) + +func TestParseWeekdays(t *testing.T) { + tests := []struct { + input string + result string + }{ + {"0,4", "Sun---------Thu------"}, + {"su,mo,tu", "SunMonTue------------"}, + {"sunday,tu,thu", "Sun---Tue---Thu------"}, + {"THURSDAY", "------------Thu------"}, + {"we,WED,WeDnEsDaY", "---------Wed---------"}, + {"", "---------------------"}, + {",,,", "---------------------"}, + } + + for _, tst := range tests { + res, err := parseWeekdays(strings.Split(tst.input, ",")) + if err != nil { + t.Errorf("Received error for input %s: %v", tst.input, err) + } else if res.String() != tst.result { + t.Errorf("Test %s: Expected %s got %s", tst.input, tst.result, res.String()) + } + } +} + +func TestParseWeekdaysErrors(t *testing.T) { + tests := []string{ + "15", + "-8", + "8", + "mon,tue,wed,fridayyyy", + } + + for _, tst := range tests { + _, err := parseWeekdays(strings.Split(tst, ",")) + if err == nil { + t.Errorf("Expected to receive error for input %s", tst) + } + } +} diff --git a/pkg/timewindow/timewindow.go b/pkg/timewindow/timewindow.go new file mode 100644 index 0000000..6ac1a0e --- /dev/null +++ b/pkg/timewindow/timewindow.go @@ -0,0 +1,68 @@ +package timewindow + +import ( + "fmt" + "time" +) + +// TimeWindow specifies a schedule of days and times. +type TimeWindow struct { + days weekdays + location *time.Location + startTime time.Time + endTime time.Time +} + +// New creates a TimeWindow instance based on string inputs specifying a schedule. +func New(days []string, startTime, endTime, location string) (*TimeWindow, error) { + tw := &TimeWindow{} + + var err error + if tw.days, err = parseWeekdays(days); err != nil { + return nil, err + } + + if tw.location, err = time.LoadLocation(location); err != nil { + return nil, err + } + + if tw.startTime, err = parseTime(startTime, tw.location); err != nil { + return nil, err + } + + if tw.endTime, err = parseTime(endTime, tw.location); err != nil { + return nil, err + } + + return tw, nil +} + +// Contains determines whether the specified time is within this time window. +func (tw *TimeWindow) Contains(t time.Time) bool { + loctime := t.In(tw.location) + if !tw.days.Contains(loctime.Weekday()) { + return false + } + + start := time.Date(loctime.Year(), loctime.Month(), loctime.Day(), tw.startTime.Hour(), tw.startTime.Minute(), tw.startTime.Second(), 0, tw.location) + end := time.Date(loctime.Year(), loctime.Month(), loctime.Day(), tw.endTime.Hour(), tw.endTime.Minute(), tw.endTime.Second(), 1e9-1, tw.location) + + return (loctime.After(start) || loctime.Equal(start)) && (loctime.Before(end) || loctime.Equal(end)) +} + +// String returns a string representation of this time window. +func (tw *TimeWindow) String() string { + return fmt.Sprintf("%s between %02d:%02d and %02d:%02d %s", tw.days.String(), tw.startTime.Hour(), tw.startTime.Minute(), tw.endTime.Hour(), tw.endTime.Minute(), tw.location.String()) +} + +// parseTime tries to parse a time with several formats. +func parseTime(s string, loc *time.Location) (time.Time, error) { + fmts := []string{"15:04", "15:04:05", "03:04pm", "15", "03pm", "3pm"} + for _, f := range fmts { + if t, err := time.ParseInLocation(f, s, loc); err == nil { + return t, nil + } + } + + return time.Now(), fmt.Errorf("Invalid time format: %s", s) +} diff --git a/pkg/timewindow/timewindow_test.go b/pkg/timewindow/timewindow_test.go new file mode 100644 index 0000000..a629a45 --- /dev/null +++ b/pkg/timewindow/timewindow_test.go @@ -0,0 +1,60 @@ +package timewindow + +import ( + "strings" + "testing" + "time" +) + +func TestTimeWindows(t *testing.T) { + type testcase struct { + time string + result bool + } + + tests := []struct { + days string + start string + end string + loc string + cases []testcase + }{ + {"mon,tue,wed,thu,fri", "9am", "5pm", "America/Los_Angeles", []testcase{ + {"2019/04/04 00:49 PDT", false}, + {"2019/04/05 08:59 PDT", false}, + {"2019/04/05 9:01 PDT", true}, + {"2019/03/31 10:00 PDT", false}, + {"2019/04/04 12:00 PDT", true}, + {"2019/04/04 11:59 UTC", false}, + }}, + {"mon,we,fri", "10:01", "11:30am", "America/Los_Angeles", []testcase{ + {"2019/04/05 10:30 PDT", true}, + {"2019/04/06 10:30 PDT", false}, + {"2019/04/07 10:30 PDT", false}, + {"2019/04/08 10:30 PDT", true}, + {"2019/04/09 10:30 PDT", false}, + {"2019/04/10 10:30 PDT", true}, + {"2019/04/11 10:30 PDT", false}, + }}, + {"mo,tu,we,th,fr", "00:00", "23:59:59", "UTC", []testcase{ + {"2019/04/18 00:00 UTC", true}, + {"2019/04/18 23:59 UTC", true}, + }}, + } + + for i, tst := range tests { + tw, err := New(strings.Split(tst.days, ","), tst.start, tst.end, tst.loc) + if err != nil { + t.Errorf("Test [%d] failed to create TimeWindow: %v", i, err) + } + + for _, cas := range tst.cases { + tm, err := time.ParseInLocation("2006/01/02 15:04 MST", cas.time, tw.location) + if err != nil { + t.Errorf("Failed to parse time \"%s\": %v", cas.time, err) + } else if cas.result != tw.Contains(tm) { + t.Errorf("(%s) contains (%s) didn't match expected result of %v", tw.String(), cas.time, cas.result) + } + } + } +}