github/kured
1
0
Fork 0
mirror of https://github.com/kubereboot/kured.git synced 2026-08-19 11:26:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into master

Browse Source
This commit is contained in:
Nighthawk22
2019-10-28 10:56:13 +01:00
committed by GitHub
parent 7944fd2639 4beddb5338
commit 5c21206bdb
8 changed files with 322 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+28
View File
@@ -6,6 +6,7 @@
* [Installation](#installation)
* [Configuration](#configuration)
* [Reboot Sentinel File & Period](#reboot-sentinel-file-&-period)
* [Setting a schedule](#setting-a-schedule)
* [Blocking Reboots via Alerts](#blocking-reboots-via-alerts)
* [Blocking Reboots via Pods](#blocking-reboots-via-pods)
* [Prometheus Metrics](#prometheus-metrics)
@@ -74,14 +75,18 @@ Flags:
--blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots
--ds-name string name of daemonset on which to place lock (default "kured")
--ds-namespace string namespace containing daemonset on which to place lock (default "kube-system")
--end-time string only reboot before this time of day (default "23:59")
-h, --help help for kured
--lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock")
--period duration reboot check period (default 1h0m0s)
--prometheus-url string Prometheus instance to probe for active alerts
--reboot-days strings only reboot on these days (default [su,mo,tu,we,th,fr,sa])
--reboot-sentinel string path to file whose existence signals need to reboot (default "/var/run/reboot-required")
--slack-channel string slack channel for reboot notfications
--slack-hook-url string slack hook URL for reboot notfications
--slack-username string slack username for reboot notfications (default "kured")
--start-time string only reboot after this time of day (default "0:00")
--time-zone string use this timezone to calculate allowed reboot time (default "UTC")
```
### Reboot Sentinel File & Period
@@ -92,6 +97,29 @@ values with `--reboot-sentinel` and `--period`. Each replica of the
daemon uses a random offset derived from the period on startup so that
nodes don't all contend for the lock simultaneously.
### Setting a schedule
By default, kured will reboot any time it detects the sentinel, but this
may cause reboots during odd hours. While service disruption does not
normally occur, anything is possible and operators may want to restrict
reboots to predictable schedules. Use `--reboot-days`, `--start-time`,
`--end-time`, and `--time-zone` to set a schedule. For example, business
hours on the west coast USA can be specified with:
```
--reboot-days mon,tue,wed,thu,fri
--start-time 9am
--end-time 5pm
--time-zone America/Los_Angeles
```
Times can be formatted in numerous ways, including `5pm`, `5:00pm` `17:00`,
and `17`. `--time-zone` represents a Go `time.Location`, and can be `UTC`,
`Local`, or any entry in the standard Linux tz database.
Note that when using smaller time windows, you should consider shortening
the sentinel check period (`--period`).
### Blocking Reboots via Alerts
You may find it desirable to block automatic node reboots when there
cmd/kured/Dockerfile
+1 -1
View File
@@ -1,5 +1,5 @@
FROM alpine:3.8
RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/*
RUN apk update && apk add ca-certificates tzdata && rm -rf /var/cache/apk/*
# NB: you may need to update RBAC permissions when upgrading kubectl - see kured-rbac.yaml for details
ADD https://storage.googleapis.com/kubernetes-release/release/v1.14.1/bin/linux/amd64/kubectl /usr/bin/kubectl
RUN chmod 0755 /usr/bin/kubectl
cmd/kured/main.go
+24 -3
View File
@@ -21,6 +21,7 @@ import (
"github.com/weaveworks/kured/pkg/daemonsetlock"
"github.com/weaveworks/kured/pkg/delaytick"
"github.com/weaveworks/kured/pkg/notifications/slack"
"github.com/weaveworks/kured/pkg/timewindow"
)
var (
@@ -39,6 +40,11 @@ var (
slackChannel string
podSelectors []string
rebootDays []string
rebootStart string
rebootEnd string
timezone string
// Metrics
rebootRequiredGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Subsystem: "kured",
@@ -82,6 +88,15 @@ func main() {
rootCmd.PersistentFlags().StringArrayVar(&podSelectors, "blocking-pod-selector", nil,
"label selector identifying pods whose presence should prevent reboots")
rootCmd.PersistentFlags().StringSliceVar(&rebootDays, "reboot-days", timewindow.EveryDay,
"schedule reboot on these days")
rootCmd.PersistentFlags().StringVar(&rebootStart, "start-time", "0:00",
"schedule reboot only after this time of day")
rootCmd.PersistentFlags().StringVar(&rebootEnd, "end-time", "23:59:59",
"schedule reboot only before this time of day")
rootCmd.PersistentFlags().StringVar(&timezone, "time-zone", "UTC",
"use this timezone for schedule inputs")
if err := rootCmd.Execute(); err != nil {
log.Fatal(err)
}
@@ -268,7 +283,7 @@ type nodeMeta struct {
Unschedulable bool `json:"unschedulable"`
}
func rebootAsRequired(nodeID string) {
func rebootAsRequired(nodeID string, window *timewindow.TimeWindow) {
config, err := rest.InClusterConfig()
if err != nil {
log.Fatal(err)
@@ -292,7 +307,7 @@ func rebootAsRequired(nodeID string) {
source := rand.NewSource(time.Now().UnixNano())
tick := delaytick.New(source, period)
for _ = range tick {
if rebootRequired() && !rebootBlocked(client, nodeID) {
if window.Contains(time.Now()) && rebootRequired() && !rebootBlocked(client, nodeID) {
node, err := client.CoreV1().Nodes().Get(nodeID, metav1.GetOptions{})
if err != nil {
log.Fatal(err)
@@ -321,12 +336,18 @@ func root(cmd *cobra.Command, args []string) {
log.Fatal("KURED_NODE_ID environment variable required")
}
window, err := timewindow.New(rebootDays, rebootStart, rebootEnd, timezone)
if err != nil {
log.Fatalf("Failed to build time window: %v", err)
}
log.Infof("Node ID: %s", nodeID)
log.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation)
log.Infof("Reboot Sentinel: %s every %v", rebootSentinel, period)
log.Infof("Blocking Pod Selectors: %v", podSelectors)
log.Infof("Reboot on: %v", window)
go rebootAsRequired(nodeID)
go rebootAsRequired(nodeID, window)
go maintainRebootRequiredMetric(nodeID)
http.Handle("/metrics", promhttp.Handler())
kured-ds.yaml
+4
View File
@@ -51,10 +51,14 @@ spec:
# - --blocking-pod-selector=...
# - --ds-name=kured
# - --ds-namespace=kube-system
# - --end-time=23:59:59
# - --lock-annotation=weave.works/kured-node-lock
# - --period=1h
# - --prometheus-url=http://prometheus.monitoring.svc.cluster.local
# - --reboot-days=sun,mon,tue,wed,thu,fri,sat
# - --reboot-sentinel=/var/run/reboot-required
# - --slack-hook-url=https://hooks.slack.com/...
# - --slack-username=prod
# - --slack-channel=alerting
# - --start-time=0:00
# - --time-zone=UTC
pkg/timewindow/days.go
+91
View File
@@ -0,0 +1,91 @@
package timewindow
import (
"fmt"
"strconv"
"strings"
"time"
)
var EveryDay = []string{"su", "mo", "tu", "we", "th", "fr", "sa"}
// dayStrings maps day strings to time.Weekdays
var dayStrings = map[string]time.Weekday{
"su": time.Sunday,
"sun": time.Sunday,
"sunday": time.Sunday,
"mo": time.Monday,
"mon": time.Monday,
"monday": time.Monday,
"tu": time.Tuesday,
"tue": time.Tuesday,
"tuesday": time.Tuesday,
"we": time.Wednesday,
"wed": time.Wednesday,
"wednesday": time.Wednesday,
"th": time.Thursday,
"thu": time.Thursday,
"thursday": time.Thursday,
"fr": time.Friday,
"fri": time.Friday,
"friday": time.Friday,
"sa": time.Saturday,
"sat": time.Saturday,
"saturday": time.Saturday,
}
type weekdays uint32
// parseWeekdays creates a set of weekdays from a string slice
func parseWeekdays(days []string) (weekdays, error) {
var result uint32
for _, day := range days {
if len(day) == 0 {
continue
}
weekday, err := parseWeekday(day)
if err != nil {
return weekdays(0), err
}
result |= 1 << uint32(weekday)
}
return weekdays(result), nil
}
// Contains returns true if the specified weekday is a member of this set.
func (w weekdays) Contains(day time.Weekday) bool {
return uint32(w)&(1<<uint32(day)) != 0
}
// String returns a string representation of the set of weekdays.
func (w weekdays) String() string {
var b strings.Builder
for i := uint32(0); i < 7; i++ {
if uint32(w)&(1<<i) != 0 {
b.WriteString(time.Weekday(i).String()[0:3])
} else {
b.WriteString("---")
}
}
return b.String()
}
func parseWeekday(day string) (time.Weekday, error) {
if n, err := strconv.Atoi(day); err == nil {
if n >= 0 && n < 7 {
return time.Weekday(n), nil
} else {
return time.Sunday, fmt.Errorf("Invalid weekday, number out of range: %s", day)
}
}
if weekday, ok := dayStrings[strings.ToLower(day)]; ok {
return weekday, nil
} else {
return time.Sunday, fmt.Errorf("Invalid weekday: %s", day)
}
}
pkg/timewindow/days_test.go
+46
View File
@@ -0,0 +1,46 @@
package timewindow
import (
"strings"
"testing"
)
func TestParseWeekdays(t *testing.T) {
tests := []struct {
input string
result string
}{
{"0,4", "Sun---------Thu------"},
{"su,mo,tu", "SunMonTue------------"},
{"sunday,tu,thu", "Sun---Tue---Thu------"},
{"THURSDAY", "------------Thu------"},
{"we,WED,WeDnEsDaY", "---------Wed---------"},
{"", "---------------------"},
{",,,", "---------------------"},
}
for _, tst := range tests {
res, err := parseWeekdays(strings.Split(tst.input, ","))
if err != nil {
t.Errorf("Received error for input %s: %v", tst.input, err)
} else if res.String() != tst.result {
t.Errorf("Test %s: Expected %s got %s", tst.input, tst.result, res.String())
}
}
}
func TestParseWeekdaysErrors(t *testing.T) {
tests := []string{
"15",
"-8",
"8",
"mon,tue,wed,fridayyyy",
}
for _, tst := range tests {
_, err := parseWeekdays(strings.Split(tst, ","))
if err == nil {
t.Errorf("Expected to receive error for input %s", tst)
}
}
}
pkg/timewindow/timewindow.go
+68
View File
@@ -0,0 +1,68 @@
package timewindow
import (
"fmt"
"time"
)
// TimeWindow specifies a schedule of days and times.
type TimeWindow struct {
days weekdays
location *time.Location
startTime time.Time
endTime time.Time
}
// New creates a TimeWindow instance based on string inputs specifying a schedule.
func New(days []string, startTime, endTime, location string) (*TimeWindow, error) {
tw := &TimeWindow{}
var err error
if tw.days, err = parseWeekdays(days); err != nil {
return nil, err
}
if tw.location, err = time.LoadLocation(location); err != nil {
return nil, err
}
if tw.startTime, err = parseTime(startTime, tw.location); err != nil {
return nil, err
}
if tw.endTime, err = parseTime(endTime, tw.location); err != nil {
return nil, err
}
return tw, nil
}
// Contains determines whether the specified time is within this time window.
func (tw *TimeWindow) Contains(t time.Time) bool {
loctime := t.In(tw.location)
if !tw.days.Contains(loctime.Weekday()) {
return false
}
start := time.Date(loctime.Year(), loctime.Month(), loctime.Day(), tw.startTime.Hour(), tw.startTime.Minute(), tw.startTime.Second(), 0, tw.location)
end := time.Date(loctime.Year(), loctime.Month(), loctime.Day(), tw.endTime.Hour(), tw.endTime.Minute(), tw.endTime.Second(), 1e9-1, tw.location)
return (loctime.After(start) || loctime.Equal(start)) && (loctime.Before(end) || loctime.Equal(end))
}
// String returns a string representation of this time window.
func (tw *TimeWindow) String() string {
return fmt.Sprintf("%s between %02d:%02d and %02d:%02d %s", tw.days.String(), tw.startTime.Hour(), tw.startTime.Minute(), tw.endTime.Hour(), tw.endTime.Minute(), tw.location.String())
}
// parseTime tries to parse a time with several formats.
func parseTime(s string, loc *time.Location) (time.Time, error) {
fmts := []string{"15:04", "15:04:05", "03:04pm", "15", "03pm", "3pm"}
for _, f := range fmts {
if t, err := time.ParseInLocation(f, s, loc); err == nil {
return t, nil
}
}
return time.Now(), fmt.Errorf("Invalid time format: %s", s)
}
pkg/timewindow/timewindow_test.go
+60
View File
@@ -0,0 +1,60 @@
package timewindow
import (
"strings"
"testing"
"time"
)
func TestTimeWindows(t *testing.T) {
type testcase struct {
time string
result bool
}
tests := []struct {
days string
start string
end string
loc string
cases []testcase
}{
{"mon,tue,wed,thu,fri", "9am", "5pm", "America/Los_Angeles", []testcase{
{"2019/04/04 00:49 PDT", false},
{"2019/04/05 08:59 PDT", false},
{"2019/04/05 9:01 PDT", true},
{"2019/03/31 10:00 PDT", false},
{"2019/04/04 12:00 PDT", true},
{"2019/04/04 11:59 UTC", false},
}},
{"mon,we,fri", "10:01", "11:30am", "America/Los_Angeles", []testcase{
{"2019/04/05 10:30 PDT", true},
{"2019/04/06 10:30 PDT", false},
{"2019/04/07 10:30 PDT", false},
{"2019/04/08 10:30 PDT", true},
{"2019/04/09 10:30 PDT", false},
{"2019/04/10 10:30 PDT", true},
{"2019/04/11 10:30 PDT", false},
}},
{"mo,tu,we,th,fr", "00:00", "23:59:59", "UTC", []testcase{
{"2019/04/18 00:00 UTC", true},
{"2019/04/18 23:59 UTC", true},
}},
}
for i, tst := range tests {
tw, err := New(strings.Split(tst.days, ","), tst.start, tst.end, tst.loc)
if err != nil {
t.Errorf("Test [%d] failed to create TimeWindow: %v", i, err)
}
for _, cas := range tst.cases {
tm, err := time.ParseInLocation("2006/01/02 15:04 MST", cas.time, tw.location)
if err != nil {
t.Errorf("Failed to parse time \"%s\": %v", cas.time, err)
} else if cas.result != tw.Contains(tm) {
t.Errorf("(%s) contains (%s) didn't match expected result of %v", tw.String(), cas.time, cas.result)
}
}
}
}