From 65292983f2ed34171345d586d017c2a39d21c11a Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Thu, 17 Dec 2020 13:36:02 -0500 Subject: [PATCH 01/25] Add force-reboot after force-timeout duration has been exceeded --- README.md | 3 +++ charts/kured/Chart.yaml | 2 +- charts/kured/README.md | 3 +++ charts/kured/templates/daemonset.yaml | 9 +++++++++ charts/kured/values.minikube.yaml | 3 +++ charts/kured/values.yaml | 3 +++ cmd/kured/main.go | 22 ++++++++++++++++++++-- 7 files changed, 42 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 44836ac..5eaba7f 100644 --- a/README.md +++ b/README.md @@ -83,9 +83,12 @@ The following arguments can be passed to kured via the daemonset pod template: Flags: --alert-filter-regexp regexp.Regexp alert names to ignore when checking for active alerts --blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots + --drain-grace-period int period of time to wait for node drain in seconds (default -1) --ds-name string name of daemonset on which to place lock (default "kured") --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") --end-time string schedule reboot only before this time of day (default "23:59:59") + --force-reboot bool force a reboot even if the drain is still running (default false) + --force-timeout duration total drain timeout, only applies when force-reboot is enabled (default 30m) -h, --help help for kured --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") --lock-ttl duration expire lock annotation after this duration (default: 0, disabled) diff --git a/charts/kured/Chart.yaml b/charts/kured/Chart.yaml index 1f3c240..0be27a2 100644 --- a/charts/kured/Chart.yaml +++ b/charts/kured/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: "1.6.1" description: A Helm chart for kured name: kured -version: 2.4.2 +version: 2.4.3 home: https://github.com/weaveworks/kured maintainers: - name: ckotzbauer diff --git a/charts/kured/README.md b/charts/kured/README.md index 3b4b7b8..cf43f20 100644 --- a/charts/kured/README.md +++ b/charts/kured/README.md @@ -50,6 +50,7 @@ The following changes have been made compared to the stable chart: | `configuration.endTime` | cli-parameter `--end-time` | `""` | | `configuration.lockAnnotation` | cli-parameter `--lock-annotation` | `""` | | `configuration.period` | cli-parameter `--period` | `""` | +| `configuration.drainGracePeriod` | cli-parameter `--drain-grace-period` | `""` | | `configuration.prometheusUrl` | cli-parameter `--prometheus-url` | `""` | | `configuration.rebootDays` | Array of days for multiple cli-parameters `--reboot-days` | `[]` | | `configuration.rebootSentinel` | cli-parameter `--reboot-sentinel` | `""` | @@ -61,6 +62,8 @@ The following changes have been made compared to the stable chart: | `configuration.startTime` | cli-parameter `--start-time` | `""` | | `configuration.timeZone` | cli-parameter `--time-zone` | `""` | | `configuration.annotateNodes` | cli-parameter `--annotate-nodes` | `false` | +| `configuration.forceReboot` | cli-parameter `--force-reboot` | `false` | +| `configuration.forceTimeout` | cli-parameter `--force-timeout` | `""` | | `rbac.create` | Create RBAC roles | `true` | | `serviceAccount.create` | Create a service account | `true` | | `serviceAccount.name` | Service account name to create (or use if `serviceAccount.create` is false) | (chart fullname) | diff --git a/charts/kured/templates/daemonset.yaml b/charts/kured/templates/daemonset.yaml index c2a6b97..4ee9f69 100644 --- a/charts/kured/templates/daemonset.yaml +++ b/charts/kured/templates/daemonset.yaml @@ -70,6 +70,9 @@ spec: {{- if .Values.configuration.period }} - --period={{ .Values.configuration.period }} {{- end }} + {{- if .Values.configuration.drainGracePeriod }} + - --drain-grace-period={{ .Values.configuration.drainGracePeriod }} + {{- end }} {{- if .Values.configuration.prometheusUrl }} - --prometheus-url={{ .Values.configuration.prometheusUrl }} {{- end }} @@ -103,6 +106,12 @@ spec: {{- if .Values.configuration.annotateNodes }} - --annotate-nodes={{ .Values.configuration.annotateNodes }} {{- end }} + {{- if .Values.configuration.forceReboot }} + - --force-reboot + {{- end }} + {{- if .Values.configuration.forceTimeout }} + - --force-timeout={{ .Values.configuration.forceTimeout }} + {{- end }} {{- range $key, $value := .Values.extraArgs }} {{- if $value }} - --{{ $key }}={{ $value }} diff --git a/charts/kured/values.minikube.yaml b/charts/kured/values.minikube.yaml index cf4a554..dcc5297 100644 --- a/charts/kured/values.minikube.yaml +++ b/charts/kured/values.minikube.yaml @@ -9,6 +9,7 @@ configuration: # endTime: "" # only reboot before this time of day (default "23:59") # lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") period: "1m" # reboot check period (default 1h0m0s) + # drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1) # prometheusUrl: "" # Prometheus instance to probe for active alerts # rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) # rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -20,3 +21,5 @@ configuration: # startTime: "" # only reboot after this time of day (default "0:00") # timeZone: "" # time-zone to use (valid zones from "time" golang package) # annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations + # forceReboot: false # force a reboot even if the drain is still running (default false) + # forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m) diff --git a/charts/kured/values.yaml b/charts/kured/values.yaml index eb0022b..29f70c9 100644 --- a/charts/kured/values.yaml +++ b/charts/kured/values.yaml @@ -28,6 +28,7 @@ configuration: endTime: "" # only reboot before this time of day (default "23:59") lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") period: "" # reboot check period (default 1h0m0s) + drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1) prometheusUrl: "" # Prometheus instance to probe for active alerts rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -39,6 +40,8 @@ configuration: startTime: "" # only reboot after this time of day (default "0:00") timeZone: "" # time-zone to use (valid zones from "time" golang package) annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations + forceReboot: false # force a reboot even if the drain is still running (default false) + forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m) rbac: create: true diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 3810145..ddfe09f 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -38,7 +38,11 @@ var ( version = "unreleased" // Command line flags + forceDrain bool + forceReboot bool + forceTimeout time.Duration period time.Duration + drainGracePeriod int dsNamespace string dsName string lockAnnotation string @@ -91,6 +95,12 @@ func main() { PreRun: flagCheck, Run: root} + rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false, + "enable/disable force reboot") + rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1, + "drain grace period in seconds") + rootCmd.PersistentFlags().DurationVar(&forceTimeout, "force-timeout", time.Minute*30, + "total timeout which only applies when force-reboot is set to true") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, "sentinel check period") rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system", @@ -333,19 +343,27 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { drainer := &kubectldrain.Helper{ Client: client, - GracePeriodSeconds: -1, + Ctx: context.Background(), + GracePeriodSeconds: drainGracePeriod, Force: true, DeleteEmptyDirData: true, IgnoreAllDaemonSets: true, ErrOut: os.Stderr, Out: os.Stdout, } + if forceReboot { + drainer.Timeout = forceTimeout + } + if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { log.Fatalf("Error cordonning %s: %v", nodename, err) } if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil { - log.Fatalf("Error draining %s: %v", nodename, err) + if !forceReboot { + log.Fatalf("Error draining %s: %v", nodename, err) + } + log.Errorf("Error draining %s: %v, continuing with reboot anyway", nodename, err) } } From 390f6e9f9930b6a805baef6a2dbeb7f5544b0a2f Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Mar 2021 16:14:19 -0800 Subject: [PATCH 02/25] chore: retry daemonset get operations --- pkg/daemonsetlock/daemonsetlock.go | 41 ++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/pkg/daemonsetlock/daemonsetlock.go b/pkg/daemonsetlock/daemonsetlock.go index eb038be..631d0df 100644 --- a/pkg/daemonsetlock/daemonsetlock.go +++ b/pkg/daemonsetlock/daemonsetlock.go @@ -6,11 +6,18 @@ import ( "fmt" "time" + v1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" ) +const ( + k8sAPICallRetrySleep = 5 * time.Second // How much time to wait in between retrying a k8s API call + k8sAPICallRetryTimeout = 5 * time.Minute // How long to wait until we determine that the k8s API is definitively unavailable +) + // DaemonSetLock holds all necessary information to do actions // on the kured ds which holds lock info through annotations. type DaemonSetLock struct { @@ -34,11 +41,11 @@ func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation strin } // Acquire attempts to annotate the kured daemonset with lock info from instantiated DaemonSetLock using client-go -func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acquired bool, owner string, err error) { +func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (bool, string, error) { for { - ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{}) + ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout) if err != nil { - return false, "", err + return false, "", fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err) } valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] @@ -78,10 +85,10 @@ func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acqu } // Test attempts to check the kured daemonset lock status (existence, expiry) from instantiated DaemonSetLock using client-go -func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) { - ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{}) +func (dsl *DaemonSetLock) Test(metadata interface{}) (bool, error) { + ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout) if err != nil { - return false, err + return false, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err) } valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] @@ -102,9 +109,9 @@ func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) { // Release attempts to remove the lock data from the kured ds annotations using client-go func (dsl *DaemonSetLock) Release() error { for { - ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{}) + ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout) if err != nil { - return err + return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err) } valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] @@ -137,6 +144,24 @@ func (dsl *DaemonSetLock) Release() error { } } +// GetDaemonSet returns the named DaemonSet resource from the DaemonSetLock's configured client +func (dsl *DaemonSetLock) GetDaemonSet(sleep, timeout time.Duration) (*v1.DaemonSet, error) { + var ds *v1.DaemonSet + var lastError error + err := wait.PollImmediate(sleep, timeout, func() (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + if ds, lastError = dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(ctx, dsl.name, metav1.GetOptions{}); lastError != nil { + return false, nil + } + return true, nil + }) + if err != nil { + return nil, fmt.Errorf("Timed out trying to get daemonset %s in namespace %s: %v", dsl.name, dsl.namespace, lastError) + } + return ds, nil +} + func ttlExpired(created time.Time, ttl time.Duration) bool { if ttl > 0 && time.Since(created) >= ttl { return true From 8db565051047dcd6857bd6c6237554661bf24d54 Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Wed, 7 Apr 2021 12:53:19 -0500 Subject: [PATCH 03/25] Refactor force-drain to be a drain-timeout in general --- README.md | 4 ++-- charts/kured/Chart.yaml | 2 +- charts/kured/README.md | 3 --- charts/kured/templates/daemonset.yaml | 9 --------- charts/kured/values.minikube.yaml | 3 --- charts/kured/values.yaml | 3 --- cmd/kured/main.go | 15 ++++++--------- 7 files changed, 9 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 5eaba7f..fd9f6ea 100644 --- a/README.md +++ b/README.md @@ -83,12 +83,12 @@ The following arguments can be passed to kured via the daemonset pod template: Flags: --alert-filter-regexp regexp.Regexp alert names to ignore when checking for active alerts --blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots - --drain-grace-period int period of time to wait for node drain in seconds (default -1) + --drain-grace-period int grace period of time for pods to wait for the node drain in seconds (default -1) --ds-name string name of daemonset on which to place lock (default "kured") --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") --end-time string schedule reboot only before this time of day (default "23:59:59") --force-reboot bool force a reboot even if the drain is still running (default false) - --force-timeout duration total drain timeout, only applies when force-reboot is enabled (default 30m) + --drain-timeout duration timeout after which the drain is aborted (default 30m) -h, --help help for kured --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") --lock-ttl duration expire lock annotation after this duration (default: 0, disabled) diff --git a/charts/kured/Chart.yaml b/charts/kured/Chart.yaml index 0be27a2..1f3c240 100644 --- a/charts/kured/Chart.yaml +++ b/charts/kured/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: "1.6.1" description: A Helm chart for kured name: kured -version: 2.4.3 +version: 2.4.2 home: https://github.com/weaveworks/kured maintainers: - name: ckotzbauer diff --git a/charts/kured/README.md b/charts/kured/README.md index cf43f20..3b4b7b8 100644 --- a/charts/kured/README.md +++ b/charts/kured/README.md @@ -50,7 +50,6 @@ The following changes have been made compared to the stable chart: | `configuration.endTime` | cli-parameter `--end-time` | `""` | | `configuration.lockAnnotation` | cli-parameter `--lock-annotation` | `""` | | `configuration.period` | cli-parameter `--period` | `""` | -| `configuration.drainGracePeriod` | cli-parameter `--drain-grace-period` | `""` | | `configuration.prometheusUrl` | cli-parameter `--prometheus-url` | `""` | | `configuration.rebootDays` | Array of days for multiple cli-parameters `--reboot-days` | `[]` | | `configuration.rebootSentinel` | cli-parameter `--reboot-sentinel` | `""` | @@ -62,8 +61,6 @@ The following changes have been made compared to the stable chart: | `configuration.startTime` | cli-parameter `--start-time` | `""` | | `configuration.timeZone` | cli-parameter `--time-zone` | `""` | | `configuration.annotateNodes` | cli-parameter `--annotate-nodes` | `false` | -| `configuration.forceReboot` | cli-parameter `--force-reboot` | `false` | -| `configuration.forceTimeout` | cli-parameter `--force-timeout` | `""` | | `rbac.create` | Create RBAC roles | `true` | | `serviceAccount.create` | Create a service account | `true` | | `serviceAccount.name` | Service account name to create (or use if `serviceAccount.create` is false) | (chart fullname) | diff --git a/charts/kured/templates/daemonset.yaml b/charts/kured/templates/daemonset.yaml index 4ee9f69..c2a6b97 100644 --- a/charts/kured/templates/daemonset.yaml +++ b/charts/kured/templates/daemonset.yaml @@ -70,9 +70,6 @@ spec: {{- if .Values.configuration.period }} - --period={{ .Values.configuration.period }} {{- end }} - {{- if .Values.configuration.drainGracePeriod }} - - --drain-grace-period={{ .Values.configuration.drainGracePeriod }} - {{- end }} {{- if .Values.configuration.prometheusUrl }} - --prometheus-url={{ .Values.configuration.prometheusUrl }} {{- end }} @@ -106,12 +103,6 @@ spec: {{- if .Values.configuration.annotateNodes }} - --annotate-nodes={{ .Values.configuration.annotateNodes }} {{- end }} - {{- if .Values.configuration.forceReboot }} - - --force-reboot - {{- end }} - {{- if .Values.configuration.forceTimeout }} - - --force-timeout={{ .Values.configuration.forceTimeout }} - {{- end }} {{- range $key, $value := .Values.extraArgs }} {{- if $value }} - --{{ $key }}={{ $value }} diff --git a/charts/kured/values.minikube.yaml b/charts/kured/values.minikube.yaml index dcc5297..cf4a554 100644 --- a/charts/kured/values.minikube.yaml +++ b/charts/kured/values.minikube.yaml @@ -9,7 +9,6 @@ configuration: # endTime: "" # only reboot before this time of day (default "23:59") # lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") period: "1m" # reboot check period (default 1h0m0s) - # drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1) # prometheusUrl: "" # Prometheus instance to probe for active alerts # rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) # rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -21,5 +20,3 @@ configuration: # startTime: "" # only reboot after this time of day (default "0:00") # timeZone: "" # time-zone to use (valid zones from "time" golang package) # annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations - # forceReboot: false # force a reboot even if the drain is still running (default false) - # forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m) diff --git a/charts/kured/values.yaml b/charts/kured/values.yaml index 29f70c9..eb0022b 100644 --- a/charts/kured/values.yaml +++ b/charts/kured/values.yaml @@ -28,7 +28,6 @@ configuration: endTime: "" # only reboot before this time of day (default "23:59") lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") period: "" # reboot check period (default 1h0m0s) - drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1) prometheusUrl: "" # Prometheus instance to probe for active alerts rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -40,8 +39,6 @@ configuration: startTime: "" # only reboot after this time of day (default "0:00") timeZone: "" # time-zone to use (valid zones from "time" golang package) annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations - forceReboot: false # force a reboot even if the drain is still running (default false) - forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m) rbac: create: true diff --git a/cmd/kured/main.go b/cmd/kured/main.go index ddfe09f..7407b09 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -38,9 +38,8 @@ var ( version = "unreleased" // Command line flags - forceDrain bool forceReboot bool - forceTimeout time.Duration + drainTimeout time.Duration period time.Duration drainGracePeriod int dsNamespace string @@ -96,11 +95,11 @@ func main() { Run: root} rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false, - "enable/disable force reboot") + "force a reboot even if the drain is still running (default false)") rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1, - "drain grace period in seconds") - rootCmd.PersistentFlags().DurationVar(&forceTimeout, "force-timeout", time.Minute*30, - "total timeout which only applies when force-reboot is set to true") + "grace period of time for pods to wait for the node drain in seconds (default -1)") + rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", time.Minute*30, + "timeout after which the drain is aborted (default: 30m)") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, "sentinel check period") rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system", @@ -350,9 +349,7 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { IgnoreAllDaemonSets: true, ErrOut: os.Stderr, Out: os.Stdout, - } - if forceReboot { - drainer.Timeout = forceTimeout + Timeout: drainTimeout, } if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { From 2400f34cc0270cdb99134101fbe694c05c5a960e Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Wed, 7 Apr 2021 14:58:21 -0500 Subject: [PATCH 04/25] Don't panic if the cordon fails and force-reboot is true --- cmd/kured/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 7407b09..71031ab 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -353,7 +353,10 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { } if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { - log.Fatalf("Error cordonning %s: %v", nodename, err) + if !forceReboot { + log.Fatalf("Error cordonning %s: %v", nodename, err) + } + log.Errorf("Error cordonning %s: %v, continuing with reboot anyway", nodename, err) } if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil { From 5a86ef40e87b784ade4aed0ed1b9713da38158d2 Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Wed, 7 Apr 2021 17:17:33 -0500 Subject: [PATCH 05/25] Update the default drain timeout to be infinite --- cmd/kured/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 71031ab..7335f41 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -98,8 +98,8 @@ func main() { "force a reboot even if the drain is still running (default false)") rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1, "grace period of time for pods to wait for the node drain in seconds (default -1)") - rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", time.Minute*30, - "timeout after which the drain is aborted (default: 30m)") + rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 0, + "timeout after which the drain is aborted (default: 0, infinite time)") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, "sentinel check period") rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system", From 25dcf3cb1286347199693b54844e78cb5944c54e Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Thu, 8 Apr 2021 09:49:44 -0500 Subject: [PATCH 06/25] Expose SkipWaitForDeleteTimeoutSeconds and explicitly return when cordonning fails --- README.md | 7 +++-- cmd/kured/main.go | 70 +++++++++++++++++++++++++---------------------- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index fd9f6ea..d5c9959 100644 --- a/README.md +++ b/README.md @@ -83,12 +83,13 @@ The following arguments can be passed to kured via the daemonset pod template: Flags: --alert-filter-regexp regexp.Regexp alert names to ignore when checking for active alerts --blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots - --drain-grace-period int grace period of time for pods to wait for the node drain in seconds (default -1) + --drain-grace-period int time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1) + --skip-wait-for-delete-timeout int when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0) --ds-name string name of daemonset on which to place lock (default "kured") --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") --end-time string schedule reboot only before this time of day (default "23:59:59") - --force-reboot bool force a reboot even if the drain is still running (default false) - --drain-timeout duration timeout after which the drain is aborted (default 30m) + --force-reboot bool force a reboot even if the drain is still running (default: false) + --drain-timeout duration timeout after which the drain is aborted (default: 0, infinite time) -h, --help help for kured --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") --lock-ttl duration expire lock annotation after this duration (default: 0, disabled) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 7335f41..45b71f8 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -38,27 +38,28 @@ var ( version = "unreleased" // Command line flags - forceReboot bool - drainTimeout time.Duration - period time.Duration - drainGracePeriod int - dsNamespace string - dsName string - lockAnnotation string - lockTTL time.Duration - prometheusURL string - preferNoScheduleTaintName string - alertFilter *regexp.Regexp - rebootSentinelFile string - rebootSentinelCommand string - notifyURL string - slackHookURL string - slackUsername string - slackChannel string - messageTemplateDrain string - messageTemplateReboot string - podSelectors []string - rebootCommand string + forceReboot bool + drainTimeout time.Duration + period time.Duration + drainGracePeriod int + skipWaitForDeleteTimeoutSeconds int + dsNamespace string + dsName string + lockAnnotation string + lockTTL time.Duration + prometheusURL string + preferNoScheduleTaintName string + alertFilter *regexp.Regexp + rebootSentinelFile string + rebootSentinelCommand string + notifyURL string + slackHookURL string + slackUsername string + slackChannel string + messageTemplateDrain string + messageTemplateReboot string + podSelectors []string + rebootCommand string rebootDays []string rebootStart string @@ -95,9 +96,11 @@ func main() { Run: root} rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false, - "force a reboot even if the drain is still running (default false)") + "force a reboot even if the drain is still running (default: false)") rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1, - "grace period of time for pods to wait for the node drain in seconds (default -1)") + "time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1)") + rootCmd.PersistentFlags().IntVar(&skipWaitForDeleteTimeoutSeconds, "skip-wait-for-delete-timeout", 0, + "when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0)") rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 0, "timeout after which the drain is aborted (default: 0, infinite time)") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, @@ -341,15 +344,16 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { } drainer := &kubectldrain.Helper{ - Client: client, - Ctx: context.Background(), - GracePeriodSeconds: drainGracePeriod, - Force: true, - DeleteEmptyDirData: true, - IgnoreAllDaemonSets: true, - ErrOut: os.Stderr, - Out: os.Stdout, - Timeout: drainTimeout, + Client: client, + Ctx: context.Background(), + GracePeriodSeconds: drainGracePeriod, + SkipWaitForDeleteTimeoutSeconds: skipWaitForDeleteTimeoutSeconds, + Force: true, + DeleteEmptyDirData: true, + IgnoreAllDaemonSets: true, + ErrOut: os.Stderr, + Out: os.Stdout, + Timeout: drainTimeout, } if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { @@ -357,6 +361,7 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { log.Fatalf("Error cordonning %s: %v", nodename, err) } log.Errorf("Error cordonning %s: %v, continuing with reboot anyway", nodename, err) + return } if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil { @@ -364,6 +369,7 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { log.Fatalf("Error draining %s: %v", nodename, err) } log.Errorf("Error draining %s: %v, continuing with reboot anyway", nodename, err) + return } } From 59d526600579d1b1741f33896630cdd4b5977241 Mon Sep 17 00:00:00 2001 From: Daniel Holbach Date: Mon, 12 Apr 2021 11:05:07 +0200 Subject: [PATCH 07/25] update to latest k8s deps of 1.20 branch Signed-off-by: Daniel Holbach --- README.md | 2 +- go.mod | 8 ++++---- go.sum | 30 +++++++++++++++--------------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 44836ac..01022ad 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ server: | kured | kubectl | k8s.io/client-go | k8s.io/apimachinery | expected kubernetes compatibility | |-------|---------|------------------|---------------------|-----------------------------------| -| main | 1.20.1 | v0.20.1 | v0.20.1 | 1.19.x, 1.20.x, 1.21.x | +| main | 1.20.5 | v0.20.5 | v0.20.5 | 1.19.x, 1.20.x, 1.21.x | | 1.6.1 | 1.19.4 | v0.19.4 | v0.19.4 | 1.18.x, 1.19.x, 1.20.x | | 1.5.1 | 1.18.8 | v0.18.8 | v0.18.8 | 1.17.x, 1.18.x, 1.19.x | | 1.4.4 | 1.17.7 | v0.17.0 | v0.17.0 | 1.16.x, 1.17.x, 1.18.x | diff --git a/go.mod b/go.mod index 5d1ae2c..0279854 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.1.3 gotest.tools/v3 v3.0.3 - k8s.io/api v0.20.1 - k8s.io/apimachinery v0.20.1 - k8s.io/client-go v0.20.1 - k8s.io/kubectl v0.20.1 + k8s.io/api v0.20.5 + k8s.io/apimachinery v0.20.5 + k8s.io/client-go v0.20.5 + k8s.io/kubectl v0.20.5 ) diff --git a/go.sum b/go.sum index a3b866b..444dfbd 100644 --- a/go.sum +++ b/go.sum @@ -873,18 +873,18 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.1 h1:ud1c3W3YNzGd6ABJlbFfKXBKXO+1KdGfcgGGNgFR03E= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/apimachinery v0.20.1 h1:LAhz8pKbgR8tUwn7boK+b2HZdt7MiTu2mkYtFMUjTRQ= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/cli-runtime v0.20.1 h1:fJhRQ9EfTpJpCqSFOAqnYLuu5aAM7yyORWZ26qW1jJc= -k8s.io/cli-runtime v0.20.1/go.mod h1:6wkMM16ZXTi7Ow3JLYPe10bS+XBnIkL6V9dmEz0mbuY= -k8s.io/client-go v0.20.1 h1:Qquik0xNFbK9aUG92pxHYsyfea5/RPO9o9bSywNor+M= -k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/component-base v0.20.1 h1:6OQaHr205NSl24t5wOF2IhdrlxZTWEZwuGlLvBgaeIg= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-helpers v0.20.1/go.mod h1:Q8trCj1zyLNdeur6pD2QvsF8d/nWVfK71YjN5+qVXy4= +k8s.io/api v0.20.5 h1:zsMTffV0Le2EiI0aKvlTHEnXGxk1HiqGRhJcCPiI7JI= +k8s.io/api v0.20.5/go.mod h1:FQjAceXnVaWDeov2YUWhOb6Yt+5UjErkp6UO3nczO1Y= +k8s.io/apimachinery v0.20.5 h1:wO/FxMVRn223rAKxnBbwCyuN96bS9MFTIvP0e/V7cps= +k8s.io/apimachinery v0.20.5/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/cli-runtime v0.20.5 h1:VIT1Y6ty6TsZaKXiyqH94U+7vR0pUpBQ1UCrTNoS+UU= +k8s.io/cli-runtime v0.20.5/go.mod h1:ihjPeQWDk7NGVIkNEvpwxA3gJvqtU+LtkDj11TvyXn4= +k8s.io/client-go v0.20.5 h1:dJGtYUvFrFGjQ+GjXEIby0gZWdlAOc0xJBJqY3VyDxA= +k8s.io/client-go v0.20.5/go.mod h1:Ee5OOMMYvlH8FCZhDsacjMlCBwetbGZETwo1OA+e6Zw= +k8s.io/code-generator v0.20.5/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/component-base v0.20.5 h1:8BZQKLJGhWrxtB7kIOEejKDtAKr1HOYvB0PZNeTyLS0= +k8s.io/component-base v0.20.5/go.mod h1:l0isoBLGyQKwRoTWbPHR6jNDd3/VqQD43cNlsjddGng= +k8s.io/component-helpers v0.20.5/go.mod h1:AzTdoPj6YAN2SUfhBX/FUUU3ntfFuse03q/VMLovEsE= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -893,9 +893,9 @@ k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kubectl v0.20.1 h1:7h1vSrL/B3hLrhlCJhbTADElPKDbx+oVUt3+QDSXxBo= -k8s.io/kubectl v0.20.1/go.mod h1:2bE0JLYTRDVKDiTREFsjLAx4R2GvUtL/mGYFXfFFMzY= -k8s.io/metrics v0.20.1/go.mod h1:JhpBE/fad3yRGsgEpiZz5FQQM5wJ18OTLkD7Tv40c0s= +k8s.io/kubectl v0.20.5 h1:/wndy8hw5TsL8G8KWPDJrtPKS8D34uSdWS0BMRmtzWs= +k8s.io/kubectl v0.20.5/go.mod h1:mlNQgyV18D4XFt5BmfSkrxQNS+arT2pXDQxxnH5lMiw= +k8s.io/metrics v0.20.5/go.mod h1:vsptOayjKWKWHvWR1vFQY++vxydzaEo/2+JC7kSDKPU= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= mvdan.cc/sh v2.6.4+incompatible/go.mod h1:IeeQbZq+x2SUGBensq/jge5lLQbS3XT2ktyp3wrt4x8= From f7b3de36a611f4d0177a2ca7b81948b97e9cadf3 Mon Sep 17 00:00:00 2001 From: Steffen Pingel Date: Tue, 13 Apr 2021 09:54:30 +0200 Subject: [PATCH 08/25] Add parameter for delaying release of lock This support throtteling of reboots across the cluster and allows rebooted nodes to reschedule pods, e.g. to synchronize replicated state before rebooting the next node. --- README.md | 5 +++++ cmd/kured/main.go | 19 +++++++++++++++++-- kured-ds.yaml | 1 + 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 01022ad..daeef20 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,11 @@ which holds lock might be killed thus annotation will stay there for ever. Using `--lock-ttl=30m` will allow other nodes to take over if TTL has expired (in this case 30min) and continue reboot process. +### Delaying Lock Release + + +Using `--lock-release-delay=30m` will cause nodes to hold the lock for the specified time frame (in this case 30min) before it is released and the reboot process continues. This can be used to throttle reboots across the cluster. + ## Building Kured now uses [Go diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 3810145..fe7a784 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -43,6 +43,7 @@ var ( dsName string lockAnnotation string lockTTL time.Duration + lockReleaseDelay time.Duration prometheusURL string preferNoScheduleTaintName string alertFilter *regexp.Regexp @@ -101,6 +102,8 @@ func main() { "annotation in which to record locking node") rootCmd.PersistentFlags().DurationVar(&lockTTL, "lock-ttl", 0, "expire lock annotation after this duration (default: 0, disabled)") + rootCmd.PersistentFlags().DurationVar(&lockReleaseDelay, "lock-release-delay", 0, + "delay lock release for this duration (default: 0, disabled)") rootCmd.PersistentFlags().StringVar(&prometheusURL, "prometheus-url", "", "Prometheus instance to probe for active alerts") rootCmd.PersistentFlags().Var(®expValue{&alertFilter}, "alert-filter-regexp", @@ -307,6 +310,12 @@ func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}, TTL time.D return true } } +func throttle(releaseDelay time.Duration) { + if releaseDelay > 0 { + log.Infof("Delaying lock release by %v", releaseDelay) + time.Sleep(releaseDelay) + } +} func release(lock *daemonsetlock.DaemonSetLock) { log.Infof("Releasing lock") @@ -436,7 +445,7 @@ func deleteNodeAnnotation(client *kubernetes.Clientset, nodeID, key string) { } } -func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []string, window *timewindow.TimeWindow, TTL time.Duration) { +func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []string, window *timewindow.TimeWindow, TTL time.Duration, releaseDelay time.Duration) { config, err := rest.InClusterConfig() if err != nil { log.Fatal(err) @@ -468,6 +477,7 @@ func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []s deleteNodeAnnotation(client, nodeID, KuredRebootInProgressAnnotation) } } + throttle(releaseDelay) release(lock) } @@ -586,6 +596,11 @@ func root(cmd *cobra.Command, args []string) { } else { log.Info("Lock TTL not set, lock will remain until being released") } + if lockReleaseDelay > 0 { + log.Infof("Lock release delay set, lock release will be delayed by: %v", lockReleaseDelay) + } else { + log.Info("Lock release delay not set, lock will be released immediately after rebooting") + } log.Infof("PreferNoSchedule taint: %s", preferNoScheduleTaintName) log.Infof("Blocking Pod Selectors: %v", podSelectors) log.Infof("Reboot schedule: %v", window) @@ -601,7 +616,7 @@ func root(cmd *cobra.Command, args []string) { hostSentinelCommand := buildHostCommand(1, sentinelCommand) hostRestartCommand := buildHostCommand(1, restartCommand) - go rebootAsRequired(nodeID, hostRestartCommand, hostSentinelCommand, window, lockTTL) + go rebootAsRequired(nodeID, hostRestartCommand, hostSentinelCommand, window, lockTTL, lockReleaseDelay) go maintainRebootRequiredMetric(nodeID, hostSentinelCommand) http.Handle("/metrics", promhttp.Handler()) diff --git a/kured-ds.yaml b/kured-ds.yaml index f4d4842..c1db53c 100644 --- a/kured-ds.yaml +++ b/kured-ds.yaml @@ -64,3 +64,4 @@ spec: # - --start-time=0:00 # - --time-zone=UTC # - --annotate-nodes=false +# - --lock-release-delay=30m From 240a669727e7e86dfb152d6e659083f25491a52d Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Tue, 13 Apr 2021 13:33:15 +0200 Subject: [PATCH 09/25] Add prometheus export metrics functional testing Without this, we can't know if the exposed prometheus metrics behave properly. This is a problem, as the only way we can evaluate the success (right now), is a compilation success or failure from kured. While this is a good start, it doesn't translate to what we claim to offer: A boolean showing if a reboot is required. This fixes it by creating a new github action workflow testing if the float64 gauge is properly showing 0 for no reboot, 1 for reboot. This is done by exposing the metrics endpoint through a node port. A helm chart change was required to have the ability to expose the service on a node port. We connect to the kind node through docker in the `tests/test-metrics.sh`, where we curl the nodeport, extract the only relevant metric, and compare it to the expected result. --- .github/workflows/on-pr.yaml | 81 ++++++++++++++++++++++++++ charts/kured/Chart.yaml | 2 +- charts/kured/ci/prometheus-values.yaml | 13 +++++ charts/kured/templates/service.yaml | 5 +- charts/kured/values.yaml | 1 + tests/kind/test-metrics.sh | 19 ++++++ 6 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 charts/kured/ci/prometheus-values.yaml create mode 100755 tests/kind/test-metrics.sh diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 6f9af20..b8359d7 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -211,3 +211,84 @@ jobs: retry_wait_seconds: 60 # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'" + + test-prom: + name: "Test prometheus with latest code from HEAD" + runs-on: ubuntu-latest + # only build with oldest and newest supported, it should be good enough. + strategy: + fail-fast: false + matrix: + kubernetes: + - "1.20" + steps: + - uses: actions/checkout@v2 + - name: Find go version + run: | + GO_VERSION=$(awk '/^go/ {print $2};' go.mod) + echo "::set-output name=version::${GO_VERSION}" + id: awk_gomod + - name: Ensure go version + uses: actions/setup-go@v2 + with: + go-version: "${{ steps.awk_gomod.outputs.version }}" + - name: Build artifacts + run: | + make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image + make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart + + - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions" + run: | + sudo bash << EOF + cp /etc/docker/daemon.json /etc/docker/daemon.json.old + echo '{}' > /etc/docker/daemon.json + systemctl restart docker || journalctl --no-pager -n 500 + systemctl status docker + EOF + + # Default name for helm/kind-action kind clusters is "chart-testing" + - name: Create 1 node kind cluster + uses: helm/kind-action@master + + - name: Preload previously built images onto kind cluster + run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing + + - name: Deploy kured on default namespace with its helm chart + run: | + # Documented in official helm doc to live on the edge + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + # Refresh bins + hash -r + helm install kured ./charts/kured/ --wait --values ./charts/kured/ci/prometheus-values.yaml + kubectl config set-context kind-chart-testing + kubectl get ds --all-namespaces + kubectl describe ds kured + + - name: Ensure kured is ready + uses: nick-invision/retry@v2.4.0 + with: + timeout_minutes: 10 + max_attempts: 10 + retry_wait_seconds: 60 + # DESIRED CURRENT READY UP-TO-DATE AVAILABLE + command: "kubectl get ds kured | grep -E 'kured.*1.*1.*1.*1.*1' " + + - name: Get metrics (healthy) + uses: nick-invision/retry@v2.4.0 + with: + timeout_minutes: 2 + max_attempts: 12 + retry_wait_seconds: 5 + command: "./tests/kind/test-metrics.sh 0" + + - name: Create reboot sentinel files + run: | + ./tests/kind/create-reboot-sentinels.sh + + - name: Get metrics (need reboot) + uses: nick-invision/retry@v2.4.0 + with: + timeout_minutes: 15 + max_attempts: 10 + retry_wait_seconds: 60 + command: "./tests/kind/test-metrics.sh 1" \ No newline at end of file diff --git a/charts/kured/Chart.yaml b/charts/kured/Chart.yaml index 1f3c240..0be27a2 100644 --- a/charts/kured/Chart.yaml +++ b/charts/kured/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: "1.6.1" description: A Helm chart for kured name: kured -version: 2.4.2 +version: 2.4.3 home: https://github.com/weaveworks/kured maintainers: - name: ckotzbauer diff --git a/charts/kured/ci/prometheus-values.yaml b/charts/kured/ci/prometheus-values.yaml new file mode 100644 index 0000000..9b418a2 --- /dev/null +++ b/charts/kured/ci/prometheus-values.yaml @@ -0,0 +1,13 @@ +# This is tested twice: +# Basic install test with chart-testing (on charts PRs) +# Functional testing in PRs (other PRs) + +service: + create: true + name: kured-prometheus-endpoint + port: 8080 + type: NodePort + nodePort: 30000 + +# Do not override the configuration: period in this, so that +# We can test prometheus exposed metrics without rebooting. diff --git a/charts/kured/templates/service.yaml b/charts/kured/templates/service.yaml index 6a01a3e..0a6e437 100644 --- a/charts/kured/templates/service.yaml +++ b/charts/kured/templates/service.yaml @@ -16,11 +16,14 @@ metadata: {{- end }} {{- end }} spec: - type: ClusterIP + type: {{ .Values.service.type }} ports: - name: metrics port: {{ .Values.service.port }} targetPort: 8080 + {{- if eq .Values.service.type "NodePort" }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} selector: {{- include "kured.matchLabels" . | nindent 4 }} {{- end }} diff --git a/charts/kured/values.yaml b/charts/kured/values.yaml index eb0022b..280e77b 100644 --- a/charts/kured/values.yaml +++ b/charts/kured/values.yaml @@ -64,6 +64,7 @@ service: port: 8080 annotations: {} name: "" + type: ClusterIP podLabels: {} diff --git a/tests/kind/test-metrics.sh b/tests/kind/test-metrics.sh new file mode 100755 index 0000000..618257e --- /dev/null +++ b/tests/kind/test-metrics.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +expected="$1" +if [[ "$expected" != "0" && "$expected" != "1" ]]; then + echo "You should give an argument to this script, the gauge value (0 or 1)" + exit 1 +fi + +HOST="${HOST:-localhost}" +PORT="${PORT:-30000}" +NODENAME="${NODENAME-chart-testing-control-plane}" + +reboot_required=$(docker exec "$NODENAME" curl "http://$HOST:$PORT/metrics" | awk '/^kured_reboot_required/{print $2}') +if [[ "$reboot_required" == "$expected" ]]; then + echo "Test success" +else + echo "Test failed" + exit 1 +fi \ No newline at end of file From 83415d0e596c8dd1d2d59cdb3fb18216c1e4bc21 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Tue, 13 Apr 2021 15:32:22 +0200 Subject: [PATCH 10/25] Reduce false positives in chart testing Without this change, the "Test helm chart (install) action" will rightfully succeed when our helm chart gets installed and has no syntax issues. However, it doesn't test if kured is properly installed. For example, the helm chart can try to install a yet unpublished image, and our test will succeed, as the syntax is still valid. This is a problem, as everything looks green, but it's not effectively working. Our other jobs are focusing on code changes, so they rightfully override the image tag, which is not what we want in this "Test helm chart" action. This fixes it by adding an extra job in the workflow, depending on the chart testing. --- .github/workflows/on-pr-charts.yaml | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/on-pr-charts.yaml b/.github/workflows/on-pr-charts.yaml index 68574cc..cbd6bb1 100644 --- a/.github/workflows/on-pr-charts.yaml +++ b/.github/workflows/on-pr-charts.yaml @@ -39,3 +39,35 @@ jobs: - name: Run chart tests run: ct ${{ matrix.test-action }} --config .github/ct.yaml + + # This doesn't re-use the ct actions, due to many limitations (auto tear down, no real testing) + deploy-chart: + name: "Functional test of helm chart with pre-published images" + runs-on: ubuntu-latest + needs: test-chart + steps: + - uses: actions/checkout@v2 + + # Default name for helm/kind-action kind clusters is "chart-testing" + - name: Create 1 node kind cluster + uses: helm/kind-action@master + + - name: Deploy kured on default namespace with its helm chart + run: | + # Documented in official helm doc to live on the edge + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + # Refresh bins + hash -r + helm install kured ./charts/kured/ --set configuration.period=1m --wait + kubectl config set-context kind-chart-testing + kubectl get ds --all-namespaces + kubectl describe ds kured + + - name: Test if successful deploy + uses: nick-invision/retry@v2.4.0 + with: + timeout_minutes: 10 + max_attempts: 10 + retry_wait_seconds: 10 + # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size + command: "kubectl get ds kured | grep -E 'kured.*1.*1.*1.*1.*1'" \ No newline at end of file From 0b759a9ff613bd02a74319a8171f0febc82427c4 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Wed, 14 Apr 2021 10:33:08 +0200 Subject: [PATCH 11/25] Update kured-ds.yaml Without this patch, it's not clear that we added command line arguments recently. This should expose our latest changes in the future released manifest. --- kured-ds.yaml | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/kured-ds.yaml b/kured-ds.yaml index f4d4842..edf1c0d 100644 --- a/kured-ds.yaml +++ b/kured-ds.yaml @@ -44,23 +44,31 @@ spec: fieldPath: spec.nodeName command: - /usr/bin/kured -# - --alert-filter-regexp=^RebootRequired$ -# - --blocking-pod-selector=runtime=long,cost=expensive -# - --blocking-pod-selector=name=temperamental -# - --blocking-pod-selector=... -# - --ds-name=kured -# - --ds-namespace=kube-system -# - --end-time=23:59:59 -# - --lock-annotation=weave.works/kured-node-lock +# - --force-reboot=false +# - --drain-grace-period=-1 +# - --skip-wait-for-delete-timeout=0 +# - --drain-timeout=0 # - --period=1h +# - --ds-namespace=kube-system +# - --ds-name=kured +# - --lock-annotation=weave.works/kured-node-lock +# - --lock-ttl=0 # - --prometheus-url=http://prometheus.monitoring.svc.cluster.local -# - --reboot-days=sun,mon,tue,wed,thu,fri,sat +# - --alert-filter-regexp=^RebootRequired$ # - --reboot-sentinel=/var/run/reboot-required +# - --prefer-no-schedule-taint="" +# - --reboot-sentinel-command="" # - --slack-hook-url=https://hooks.slack.com/... # - --slack-username=prod # - --slack-channel=alerting +# - --notify-url="" # See also shoutrrr url format # - --message-template-drain=Draining node %s # - --message-template-drain=Rebooting node %s +# - --blocking-pod-selector=runtime=long,cost=expensive +# - --blocking-pod-selector=name=temperamental +# - --blocking-pod-selector=... +# - --reboot-days=sun,mon,tue,wed,thu,fri,sat # - --start-time=0:00 +# - --end-time=23:59:59 # - --time-zone=UTC # - --annotate-nodes=false From 4a11a95b86aa15c24f11df6dbd829c71bc69f042 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Apr 2021 05:49:19 +0000 Subject: [PATCH 12/25] Bump github.com/containrrr/shoutrrr from 0.4.2 to 0.4.3 Bumps [github.com/containrrr/shoutrrr](https://github.com/containrrr/shoutrrr) from 0.4.2 to 0.4.3. - [Release notes](https://github.com/containrrr/shoutrrr/releases) - [Changelog](https://github.com/containrrr/shoutrrr/blob/main/goreleaser.yml) - [Commits](https://github.com/containrrr/shoutrrr/compare/v0.4.2...v0.4.3) Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0279854..72726ed 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/weaveworks/kured go 1.15 require ( - github.com/containrrr/shoutrrr v0.4.2 + github.com/containrrr/shoutrrr v0.4.3 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/prometheus/client_golang v1.8.0 github.com/prometheus/common v0.15.0 diff --git a/go.sum b/go.sum index 444dfbd..a1760a0 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,8 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containrrr/shoutrrr v0.4.2 h1:fAoWgO2DLbHeE3aMpeLJm/eZqZ/7hZ5aGoJAJZqGeOA= -github.com/containrrr/shoutrrr v0.4.2/go.mod h1:zqL2BvfC1W4FujrT4b3/ZCLxvD+uoeEpBL7rg9Dqpbg= +github.com/containrrr/shoutrrr v0.4.3 h1:7AHxRWFrWJJuVTfhooorVKIHOrG+v3mAk3qBkQUkBoU= +github.com/containrrr/shoutrrr v0.4.3/go.mod h1:zqL2BvfC1W4FujrT4b3/ZCLxvD+uoeEpBL7rg9Dqpbg= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 7b44fd2eb87eb356ca17693c1b777ff3a39ed370 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Apr 2021 05:48:57 +0000 Subject: [PATCH 13/25] Bump github.com/containrrr/shoutrrr from 0.4.3 to 0.4.4 Bumps [github.com/containrrr/shoutrrr](https://github.com/containrrr/shoutrrr) from 0.4.3 to 0.4.4. - [Release notes](https://github.com/containrrr/shoutrrr/releases) - [Changelog](https://github.com/containrrr/shoutrrr/blob/main/goreleaser.yml) - [Commits](https://github.com/containrrr/shoutrrr/compare/v0.4.3...v0.4.4) Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 72726ed..5c2822e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/weaveworks/kured go 1.15 require ( - github.com/containrrr/shoutrrr v0.4.3 + github.com/containrrr/shoutrrr v0.4.4 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/prometheus/client_golang v1.8.0 github.com/prometheus/common v0.15.0 diff --git a/go.sum b/go.sum index a1760a0..5e8ff5a 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,8 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containrrr/shoutrrr v0.4.3 h1:7AHxRWFrWJJuVTfhooorVKIHOrG+v3mAk3qBkQUkBoU= -github.com/containrrr/shoutrrr v0.4.3/go.mod h1:zqL2BvfC1W4FujrT4b3/ZCLxvD+uoeEpBL7rg9Dqpbg= +github.com/containrrr/shoutrrr v0.4.4 h1:vHZ4E/76pKVY+Jyn/qhBz3X540Bn8NI5ppPHK4PyILY= +github.com/containrrr/shoutrrr v0.4.4/go.mod h1:zqL2BvfC1W4FujrT4b3/ZCLxvD+uoeEpBL7rg9Dqpbg= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 0eb318c1b2465454ac9a3f32767929ccd51e2d10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Apr 2021 08:22:04 +0000 Subject: [PATCH 14/25] Bump github.com/prometheus/client_golang from 1.8.0 to 1.10.0 Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.8.0 to 1.10.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/master/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.8.0...v1.10.0) Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 23 ++++++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 5c2822e..2fddf75 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.15 require ( github.com/containrrr/shoutrrr v0.4.4 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/prometheus/client_golang v1.8.0 - github.com/prometheus/common v0.15.0 + github.com/prometheus/client_golang v1.10.0 + github.com/prometheus/common v0.18.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.1.3 gotest.tools/v3 v3.0.3 diff --git a/go.sum b/go.sum index 5e8ff5a..d2311dd 100644 --- a/go.sum +++ b/go.sum @@ -238,8 +238,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -313,6 +314,7 @@ github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZn github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -395,6 +397,7 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= @@ -459,8 +462,8 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.8.0 h1:zvJNkoCFAnYFNC24FV8nW4JdRJ3GIFcLbg65lL/JDcw= -github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= +github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg= +github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -474,17 +477,17 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= -github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= +github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -656,6 +659,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -702,10 +706,11 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY= golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 247e6f6c700c5a8985604d627026ad697be4a401 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Tue, 27 Apr 2021 10:11:16 +0200 Subject: [PATCH 15/25] Use stable kind-action We are relying on master, which might break anytime (or in this case, moved to another branch). Instead we should rely on a stable version, and unfreeze if necessary. Dependabot helps us maintain those releases anyway. --- .github/workflows/on-pr-charts.yaml | 2 +- .github/workflows/on-pr.yaml | 6 +++--- .github/workflows/periodics-daily.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/on-pr-charts.yaml b/.github/workflows/on-pr-charts.yaml index cbd6bb1..ea5ae97 100644 --- a/.github/workflows/on-pr-charts.yaml +++ b/.github/workflows/on-pr-charts.yaml @@ -50,7 +50,7 @@ jobs: # Default name for helm/kind-action kind clusters is "chart-testing" - name: Create 1 node kind cluster - uses: helm/kind-action@master + uses: helm/kind-action@v1.1.0 - name: Deploy kured on default namespace with its helm chart run: | diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index b8359d7..27d43cc 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -122,7 +122,7 @@ jobs: # Default name for helm/kind-action kind clusters is "chart-testing" - name: Create 5 node kind cluster - uses: helm/kind-action@master + uses: helm/kind-action@v1.1.0 with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml @@ -195,7 +195,7 @@ jobs: EOF # Default name for helm/kind-action kind clusters is "chart-testing" - name: Create kind cluster - uses: helm/kind-action@master + uses: helm/kind-action@v1.1.0 with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml - name: Preload previously built images onto kind cluster @@ -248,7 +248,7 @@ jobs: # Default name for helm/kind-action kind clusters is "chart-testing" - name: Create 1 node kind cluster - uses: helm/kind-action@master + uses: helm/kind-action@v1.1.0 - name: Preload previously built images onto kind cluster run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing diff --git a/.github/workflows/periodics-daily.yaml b/.github/workflows/periodics-daily.yaml index 403420c..cc12cb3 100644 --- a/.github/workflows/periodics-daily.yaml +++ b/.github/workflows/periodics-daily.yaml @@ -104,7 +104,7 @@ jobs: # Default name for helm/kind-action kind clusters is "chart-testing" - name: Create 5 node kind cluster - uses: helm/kind-action@master + uses: helm/kind-action@v1.1.0 with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml From bb56c731bb6fe739c2dd0e989adbeb2e8aa51887 Mon Sep 17 00:00:00 2001 From: papanito Date: Tue, 27 Apr 2021 13:00:19 +0200 Subject: [PATCH 16/25] docu: update url for ms teams notifications, fixes #362 --- README.md | 53 +++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4b3bb01..1d91c16 100644 --- a/README.md +++ b/README.md @@ -3,25 +3,27 @@ -* [Introduction](#introduction) -* [Kubernetes & OS Compatibility](#kubernetes-&-os-compatibility) -* [Installation](#installation) -* [Configuration](#configuration) - * [Reboot Sentinel File & Period](#reboot-sentinel-file-&-period) - * [Setting a schedule](#setting-a-schedule) - * [Blocking Reboots via Alerts](#blocking-reboots-via-alerts) - * [Blocking Reboots via Pods](#blocking-reboots-via-pods) - * [Prometheus Metrics](#prometheus-metrics) - * [Slack Notifications](#slack-notifications) - * [Overriding Lock Configuration](#overriding-lock-configuration) -* [Operation](#operation) - * [Testing](#testing) - * [Disabling Reboots](#disabling-reboots) - * [Manual Unlock](#manual-unlock) - * [Automatic Unlock](#automatic-unlock) -* [Building](#building) -* [Frequently Asked/Anticipated Questions](#frequently-askedanticipated-questions) -* [Getting Help](#getting-help) +- [Introduction](#introduction) +- [Kubernetes & OS Compatibility](#kubernetes--os-compatibility) +- [Installation](#installation) +- [Configuration](#configuration) + - [Reboot Sentinel File & Period](#reboot-sentinel-file--period) + - [Setting a schedule](#setting-a-schedule) + - [Blocking Reboots via Alerts](#blocking-reboots-via-alerts) + - [Blocking Reboots via Pods](#blocking-reboots-via-pods) + - [Prometheus Metrics](#prometheus-metrics) + - [Notifications](#notifications) + - [Overriding Lock Configuration](#overriding-lock-configuration) +- [Operation](#operation) + - [Testing](#testing) + - [Disabling Reboots](#disabling-reboots) + - [Manual Unlock](#manual-unlock) + - [Automatic Unlock](#automatic-unlock) + - [Delaying Lock Release](#delaying-lock-release) +- [Building](#building) +- [Frequently Asked/Anticipated Questions](#frequently-askedanticipated-questions) + - [Why is there no `latest` tag on Docker Hub?](#why-is-there-no-latest-tag-on-docker-hub) +- [Getting Help](#getting-help) ## Introduction @@ -95,6 +97,7 @@ Flags: --lock-ttl duration expire lock annotation after this duration (default: 0, disabled) --message-template-drain string message template used to notify about a node being drained (default "Draining node %s") --message-template-reboot string message template used to notify about a node being rebooted (default "Rebooting node %s") + --notify-url url for reboot notifications (cannot use with --slack-hook-url flags) --period duration reboot check period (default 1h0m0s) --prefer-no-schedule-taint string Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Disabled by default. Set e.g. to "weave.works/kured-node-reboot" to enable tainting. --prometheus-url string Prometheus instance to probe for active alerts @@ -103,7 +106,7 @@ Flags: --reboot-sentinel string path to file whose existence signals need to reboot (default "/var/run/reboot-required") --reboot-sentinel-command string command for which a successful run signals need to reboot (default ""). If non-empty, sentinel file will be ignored. --slack-channel string slack channel for reboot notfications - --slack-hook-url string slack hook URL for reboot notfications + --slack-hook-url string slack hook URL for reboot notfications [deprecated in favor of --notify-url] --slack-username string slack username for reboot notfications (default "kured") --start-time string schedule reboot only after this time of day (default "0:00") --time-zone string use this timezone for schedule inputs (default "UTC") @@ -238,14 +241,16 @@ Alternatively you can use the `--message-template-drain` and `--message-template Here is the syntax: -slack: `slack://tokenA/tokenB/tokenC` +- slack: `slack://tokenA/tokenB/tokenC` (`--slack-hook-url` is deprecated but possible to use) -rocketchat: `rocketchat://[username@]rocketchat-host/token[/channel|@recipient]` +- rocketchat: `rocketchat://[username@]rocketchat-host/token[/channel|@recipient]` -teams: `teams://token-a/token-b/token-c` +- teams: `teams://tName/token-a/token-b/token-c` -Email: `smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]` + > **Attention** as the [format of the url has changed](https://github.com/containrrr/shoutrrr/issues/138) you also have to specify a `tName` + +- Email: `smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]` More details here: From 4d5ea21db3c1fcd70b063331f55429c941339378 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Apr 2021 16:16:32 +0000 Subject: [PATCH 17/25] Bump github.com/prometheus/common from 0.18.0 to 0.23.0 Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.18.0 to 0.23.0. - [Release notes](https://github.com/prometheus/common/releases) - [Commits](https://github.com/prometheus/common/compare/v0.18.0...v0.23.0) Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2fddf75..f502b68 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/containrrr/shoutrrr v0.4.4 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/prometheus/client_golang v1.10.0 - github.com/prometheus/common v0.18.0 + github.com/prometheus/common v0.23.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.1.3 gotest.tools/v3 v3.0.3 diff --git a/go.sum b/go.sum index d2311dd..f55441b 100644 --- a/go.sum +++ b/go.sum @@ -477,8 +477,9 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.23.0 h1:GXWvPYuTUenIa+BhOq/x+L/QZzCqASkVRny5KTlPDGM= +github.com/prometheus/common v0.23.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= From eca6da173c9cbb50a52230e1805ab131bf9b5b39 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Thu, 8 Apr 2021 11:10:39 +0200 Subject: [PATCH 18/25] Clarify and simplify tests Without this, we get multiple questions about our testing. This should help clarify the tests and our coverage by: - Simplifying our coverage - Documenting better the purpose of each workflow file - Documenting our testing and development activities better. --- .github/workflows/on-pr-charts.yaml | 7 +- .github/workflows/on-pr.yaml | 200 ++++++++++++++----------- .github/workflows/periodics-daily.yaml | 11 +- DEVELOPMENT.md | 91 ++++++++++- Makefile | 10 +- 5 files changed, 216 insertions(+), 103 deletions(-) diff --git a/.github/workflows/on-pr-charts.yaml b/.github/workflows/on-pr-charts.yaml index ea5ae97..011bbe7 100644 --- a/.github/workflows/on-pr-charts.yaml +++ b/.github/workflows/on-pr-charts.yaml @@ -1,5 +1,6 @@ #This is just extra testing, for lint check, and basic installation -#If those fail, no need to test the rest of the PR (github will cancel the rest of the builds) +#Those can fail earlier than functional tests (shorter tests) +# and give developer feedback soon if they didn't test themselves name: PR - charts on: pull_request: @@ -11,7 +12,7 @@ jobs: # tackling that for us. # Fail-fast ensures that if one of those matrix job fail, the other one gets cancelled. test-chart: - name: Test helm chart + name: Test helm chart changes runs-on: ubuntu-latest strategy: fail-fast: true @@ -42,7 +43,7 @@ jobs: # This doesn't re-use the ct actions, due to many limitations (auto tear down, no real testing) deploy-chart: - name: "Functional test of helm chart with pre-published images" + name: Functional test of helm chart in its current state (needs published image of the helm chart) runs-on: ubuntu-latest needs: test-chart steps: diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 27d43cc..1c9c8bb 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -84,92 +84,22 @@ jobs: with: image-name: docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} - # If the PRs don't break the behaviour in the helm chart, we can simply publish the helm charts at the time of a branch commit. - e2e-helm: - name: "Functional test of helm chart, e2e testing" + # This ensures the latest code works with the manifests built from tree. + # It is useful for two things: + # - Test manifests changes (obviously), ensuring they don't break existing clusters + # - Ensure manifests work with the latest versions even with no manifest change + # (compared to helm charts, manifests cannot easily template changes based on versions) + # Helm charts are _trailing_ releases, while manifests are done during development. + e2e-manifests: + name: End-to-End test with kured with code and manifests from HEAD runs-on: ubuntu-latest - # only build with oldest and newest supported, it should be good enough. strategy: fail-fast: false matrix: kubernetes: - "1.18" - - "1.20" - steps: - - uses: actions/checkout@v2 - - name: Find go version - run: | - GO_VERSION=$(awk '/^go/ {print $2};' go.mod) - echo "::set-output name=version::${GO_VERSION}" - id: awk_gomod - - name: Ensure go version - uses: actions/setup-go@v2 - with: - go-version: "${{ steps.awk_gomod.outputs.version }}" - - name: Build artifacts - run: | - make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image - make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart - - - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions" - run: | - sudo bash << EOF - cp /etc/docker/daemon.json /etc/docker/daemon.json.old - echo '{}' > /etc/docker/daemon.json - systemctl restart docker || journalctl --no-pager -n 500 - systemctl status docker - EOF - - # Default name for helm/kind-action kind clusters is "chart-testing" - - name: Create 5 node kind cluster - uses: helm/kind-action@v1.1.0 - with: - config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml - - - name: Preload previously built images onto kind cluster - run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing - - - name: Deploy kured on default namespace with its helm chart - run: | - # Documented in official helm doc to live on the edge - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - # Refresh bins - hash -r - helm install kured ./charts/kured/ --set configuration.period=1m - kubectl config set-context kind-chart-testing - kubectl get ds --all-namespaces - kubectl describe ds kured - - - name: Ensure kured is ready - uses: nick-invision/retry@v2.4.0 - with: - timeout_minutes: 10 - max_attempts: 10 - retry_wait_seconds: 60 - # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = 5 - command: "kubectl get ds kured | grep -E 'kured.*5.*5.*5.*5.*5' " - - - name: Create reboot sentinel files - run: | - ./tests/kind/create-reboot-sentinels.sh - - - name: Follow reboot until success - env: - DEBUG: true - run: | - ./tests/kind/follow-coordinated-reboot.sh - - # This workflow is useful when introducing new versions, to ensure our manifests - # still work (even if there might be no manifest 'code' change). - # The version used here is what hasn't been tested with the helm chart - # This should therefore be "mid version" and (optionally) next new version - deploy-manifests: - name: Deploy kured with current manifests - runs-on: ubuntu-latest - strategy: - matrix: - kubernetes: - "1.19" + - "1.20" steps: - uses: actions/checkout@v2 - name: Find go version @@ -185,6 +115,7 @@ jobs: run: | make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" manifest + - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions run: | sudo bash << EOF @@ -193,16 +124,24 @@ jobs: systemctl restart docker || journalctl --no-pager -n 500 systemctl status docker EOF + # Default name for helm/kind-action kind clusters is "chart-testing" - - name: Create kind cluster + - name: Create kind cluster with 5 nodes uses: helm/kind-action@v1.1.0 with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml + - name: Preload previously built images onto kind cluster run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing + + - name: Do not wait for an hour before detecting the rebootSentinel + run: | + sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds.yaml + - name: Install kured with kubectl run: | kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml + - name: Ensure kured is ready uses: nick-invision/retry@v2.4.0 with: @@ -212,8 +151,18 @@ jobs: # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'" - test-prom: - name: "Test prometheus with latest code from HEAD" + - name: Create reboot sentinel files + run: | + ./tests/kind/create-reboot-sentinels.sh + + - name: Follow reboot until success + env: + DEBUG: true + run: | + ./tests/kind/follow-coordinated-reboot.sh + + scenario-prom-helm: + name: Test prometheus with latest code from HEAD (=overrides image of the helm chart) runs-on: ubuntu-latest # only build with oldest and newest supported, it should be good enough. strategy: @@ -237,7 +186,7 @@ jobs: make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart - - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions" + - name: Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions run: | sudo bash << EOF cp /etc/docker/daemon.json /etc/docker/daemon.json.old @@ -288,7 +237,88 @@ jobs: - name: Get metrics (need reboot) uses: nick-invision/retry@v2.4.0 with: - timeout_minutes: 15 + timeout_minutes: 15 max_attempts: 10 retry_wait_seconds: 60 - command: "./tests/kind/test-metrics.sh 1" \ No newline at end of file + command: "./tests/kind/test-metrics.sh 1" + + + # TEMPLATE Scenario testing. + # Note: keep in mind that the helm chart's appVersion is overriden to test your HEAD of the branch, + # if you `make helm-chart`. + # This will allow you to test properly your scenario and not use an existing image which will not + # contain your feature. + + # scenario--helm: + # #example: Testing with helm chart and code from HEAD" + # name: "" + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # # You can define your own kubernetes versions. For example if your helm chart change should behave differently with different kubernetes versions. + # matrix: + # kubernetes: + # - "1.20" + # steps: + # - uses: actions/checkout@v2 + # - name: Find go version + # run: | + # GO_VERSION=$(awk '/^go/ {print $2};' go.mod) + # echo "::set-output name=version::${GO_VERSION}" + # id: awk_gomod + # - name: Ensure go version + # uses: actions/setup-go@v2 + # with: + # go-version: "${{ steps.awk_gomod.outputs.version }}" + # - name: Build artifacts + # run: | + # make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image + # make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart + # + # - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions" + # run: | + # sudo bash << EOF + # cp /etc/docker/daemon.json /etc/docker/daemon.json.old + # echo '{}' > /etc/docker/daemon.json + # systemctl restart docker || journalctl --no-pager -n 500 + # systemctl status docker + # EOF + # + # # Default name for helm/kind-action kind clusters is "chart-testing" + # - name: Create 5 node kind cluster + # uses: helm/kind-action@master + # with: + # config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml + # + # - name: Preload previously built images onto kind cluster + # run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing + # + # - name: Deploy kured on default namespace with its helm chart + # run: | + # # Documented in official helm doc to live on the edge + # curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + # # Refresh bins + # hash -r + # helm install kured ./charts/kured/ --wait --values ./charts/kured/ci/-values.yaml + # kubectl config set-context kind-chart-testing + # kubectl get ds --all-namespaces + # kubectl describe ds kured + # + # - name: Ensure kured is ready + # uses: nick-invision/retry@v2.4.0 + # with: + # timeout_minutes: 10 + # max_attempts: 10 + # retry_wait_seconds: 60 + # # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = 5 + # command: "kubectl get ds kured | grep -E 'kured.*5.*5.*5.*5.*5' " + # + # - name: Create reboot sentinel files + # run: | + # ./tests/kind/create-reboot-sentinels.sh + # + # - name: Test + # env: + # DEBUG: true + # run: | + # diff --git a/.github/workflows/periodics-daily.yaml b/.github/workflows/periodics-daily.yaml index cc12cb3..de817b3 100644 --- a/.github/workflows/periodics-daily.yaml +++ b/.github/workflows/periodics-daily.yaml @@ -68,7 +68,7 @@ jobs: image-name: docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} deploy-helm: - name: Ensure a kubernetes change didn't break our code + name: Ensure our currently released helm chart works on all kubernetes versions runs-on: ubuntu-latest # only build with oldest and newest supported, it should be good enough. strategy: @@ -88,10 +88,6 @@ jobs: uses: actions/setup-go@v2 with: go-version: "${{ steps.awk_gomod.outputs.version }}" - - name: Build artifacts - run: | - make DH_ORG="${{ github.repository_owner }}" VERSION="main" image - make DH_ORG="${{ github.repository_owner }}" VERSION="main" helm-chart - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions" run: | @@ -108,9 +104,6 @@ jobs: with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml - - name: Preload previously built images onto kind cluster - run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:main --name chart-testing - - name: Deploy kured on default namespace with its helm chart run: | # Documented in official helm doc to live on the edge @@ -139,4 +132,4 @@ jobs: env: DEBUG: true run: | - ./tests/kind/follow-coordinated-reboot.sh + ./tests/kind/follow-coordinated-reboot.sh \ No newline at end of file diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 9f9f743..4da87d6 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -13,7 +13,9 @@ you are planning to contribute code. [issues]: https://github.com/weaveworks/kured/issues [readme]: README.md -## Updating k8s support +## Regular development activities + +### Updating k8s support Whenever we want to update e.g. the `kubectl` or `client-go` dependencies, some RBAC changes might be necessary too. @@ -24,15 +26,90 @@ This is what it took to support Kubernetes 1.14: That the process can be more involved that that can be seen in +Please update our .github/workflows with the new k8s images, starting by +the creation of a .github/kind-cluster-.yaml, then updating +our workflows with the new versions. + Once you updated everything, make sure you update the support matrix on the main [README][readme] as well. -## Release testing +### Updating other dependencies + +Dependabot proposes changes in our go.mod/go.sum. +Some of those changes are covered by CI testing, some are not. + +Please make sure to test those not covered by CI (mostly the integration +with other tools) manually before merging. + +### Review periodic jobs + +We run periodic jobs (see also Automated testing section of this documentation). +Those should be monitored for failures. + +If a failure happen in periodics, something terribly wrong must have happened +(or github is failing at the creation of a kind cluster). Please monitor those +failures carefully. + +### Introducing new features + +When you introduce a new feature, the kured team expects you to have tested +your change thoroughly. If possible, include all the necessary testing in your change. + +If your change involves a user facing change (change in flags of kured for example), +please include expose your new feature in our default manifest (`kured-ds.yaml`), +as a comment. + +Do not update the helm chart directly. +Helm charts and our release manifests (see below) are our stable interfaces. +Any user facing changes will therefore have to wait for a while before being +exposed to our users. + +This also means that when you expose a new feature, you should create another PR +for your changes in `charts/` to make your feature available for our next kured version. +In this change, you can directly bump the appVersion to the next minor version. +(for example, if current appVersion is 1.6.x, make sure you update your appVersion +to 1.7.0). It allows us to have an easy view of what we land each release. + +Do not hesitate to increase the test coverage for your feature, whether it's unit +testing to full functional testing (even using helm charts) + +### Increasing test coverage + +We are welcoming any change to increase our test coverage. +See also our github issues for the label `testing`. + +### Updating helm charts + +Helm charts are continuously published. Any change in `charts/` will be immediately +pushed in production. + +## Automated testing + +Our CI is covered by github actions. +You can see their contents in .github/workflows. + +We currently run: +- go tests and lint +- shellcheck +- a check for dead links in our docs +- a security check against our base image (alpine) +- a deep functional test using our manifests on all supported k8s versions +- basic deployment using our helm chart on any chart change + +Changes in helm charts are not functionally tested on PRs. We assume that +the PRs to implement the feature are properly tested by our users and +contributors before merge. + +To test your code manually, follow the section Manual testing. + +## Manual (release) testing Before `kured` is released, we want to make sure it still works fine on the previous, current and next minor version of Kubernetes (with respect to the `client-go` & `kubectl` dependencies in use). For local testing e.g. -`minikube` or `kind` can be sufficient. +`minikube` or `kind` can be sufficient. This will allow you to catch issues +that might not have been tested in our CI, like integration with other tools, +or your specific use case. Deploy kured in your test scenario, make sure you pass the right `image`, update the e.g. `period` and `reboot-days` options, so you get immediate @@ -42,7 +119,11 @@ results, if you login to a node and run: sudo touch /var/run/reboot-required ``` -### Testing with `minikube` +### Example of golang testing + +Please run `make test`. You should have golint installed. + +### Example of testing with `minikube` A test-run with `minikube` could look like this: @@ -82,7 +163,7 @@ If all the tests ran well, kured maintainers can reach out to the Weaveworks team to get an upcoming `kured` release tested in the Dev environment for real life testing. -### Testing with `kind` +### Example of testing with `kind` A test-run with `kind` could look like this: diff --git a/Makefile b/Makefile index ca80ff3..36b259c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT: all -.PHONY: all clean image publish-image minikube-publish manifest helm-chart +.PHONY: all clean image publish-image minikube-publish manifest helm-chart test tests DH_ORG=weaveworks VERSION=$(shell git symbolic-ref --short HEAD)-$(shell git rev-parse --short HEAD) @@ -43,3 +43,11 @@ helm-chart: sed -i "s#appVersion:.*#appVersion: \"$(VERSION)\"#g" charts/kured/Chart.yaml sed -i "s#\`[0-9]*\.[0-9]*\.[0-9]*\`#\`$(VERSION)\`#g" charts/kured/README.md echo "Please bump version in charts/kured/Chart.yaml" + +test: tests + echo "Running go tests" + go test ./... + echo "Running golint on pkg" + golint ./pkg/... + echo "Running golint on cmd" + golint ./cmd/... From 5061a611a8110b39450eb69ab896b968417fca95 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 May 2021 05:17:02 +0000 Subject: [PATCH 19/25] Bump github.com/prometheus/common from 0.23.0 to 0.24.0 Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.23.0 to 0.24.0. - [Release notes](https://github.com/prometheus/common/releases) - [Commits](https://github.com/prometheus/common/compare/v0.23.0...v0.24.0) Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f502b68..a7bdce6 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/containrrr/shoutrrr v0.4.4 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/prometheus/client_golang v1.10.0 - github.com/prometheus/common v0.23.0 + github.com/prometheus/common v0.24.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.1.3 gotest.tools/v3 v3.0.3 diff --git a/go.sum b/go.sum index f55441b..c019549 100644 --- a/go.sum +++ b/go.sum @@ -478,8 +478,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.23.0 h1:GXWvPYuTUenIa+BhOq/x+L/QZzCqASkVRny5KTlPDGM= -github.com/prometheus/common v0.23.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= +github.com/prometheus/common v0.24.0 h1:aIycr3wRFxPUq8XlLQlGQ9aNXV3dFi5y62pe/SB262k= +github.com/prometheus/common v0.24.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= From d7377bff1b7853b3672b62dc2f90b4a770b6b1ec Mon Sep 17 00:00:00 2001 From: Daniel Holbach Date: Tue, 18 May 2021 09:03:11 +0200 Subject: [PATCH 20/25] update golang.org/x/crypto - break out of #349 Signed-off-by: Daniel Holbach --- go.mod | 1 + go.sum | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/go.mod b/go.mod index a7bdce6..943dce7 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/prometheus/common v0.24.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.1.3 + golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect gotest.tools/v3 v3.0.3 k8s.io/api v0.20.5 k8s.io/apimachinery v0.20.5 diff --git a/go.sum b/go.sum index c019549..8b302fe 100644 --- a/go.sum +++ b/go.sum @@ -585,6 +585,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o= +golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -648,6 +650,8 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -708,10 +712,13 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 729fa658dcc1057c44822a7954dd5318b4ff7e41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 07:50:09 +0000 Subject: [PATCH 21/25] Bump actions/stale from 3.0.18 to 3.0.19 Bumps [actions/stale](https://github.com/actions/stale) from 3.0.18 to 3.0.19. - [Release notes](https://github.com/actions/stale/releases) - [Commits](https://github.com/actions/stale/compare/v3.0.18...v3.0.19) Signed-off-by: dependabot[bot] --- .github/workflows/periodics-daily.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/periodics-daily.yaml b/.github/workflows/periodics-daily.yaml index de817b3..6e5daa1 100644 --- a/.github/workflows/periodics-daily.yaml +++ b/.github/workflows/periodics-daily.yaml @@ -25,7 +25,7 @@ jobs: steps: # Stale by default waits for 60 days before marking PR/issues as stale, and closes them after 21 days. # Do not expire the first issues that would allow the community to grow. - - uses: actions/stale@v3.0.18 + - uses: actions/stale@v3.0.19 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'This issue was automatically considered stale due to lack of activity. Please update it and/or join our slack channels to promote it, before it automatically closes (in 7 days).' From d08b42933da603c8c8221f589b54ee979e07d257 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 May 2021 05:45:01 +0000 Subject: [PATCH 22/25] Bump github.com/prometheus/common from 0.24.0 to 0.25.0 Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.24.0 to 0.25.0. - [Release notes](https://github.com/prometheus/common/releases) - [Commits](https://github.com/prometheus/common/compare/v0.24.0...v0.25.0) Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 943dce7..d0fb474 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/containrrr/shoutrrr v0.4.4 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/prometheus/client_golang v1.10.0 - github.com/prometheus/common v0.24.0 + github.com/prometheus/common v0.25.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.1.3 golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect diff --git a/go.sum b/go.sum index 8b302fe..e873127 100644 --- a/go.sum +++ b/go.sum @@ -478,8 +478,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.24.0 h1:aIycr3wRFxPUq8XlLQlGQ9aNXV3dFi5y62pe/SB262k= -github.com/prometheus/common v0.24.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= +github.com/prometheus/common v0.25.0 h1:IjJYZJCI8HZYtqA3xYwGyDzSCy1r4CA2GRh+4vdOmtE= +github.com/prometheus/common v0.25.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -583,7 +583,6 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= @@ -648,7 +647,6 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= From 870329c7b45c6446444fd231f0318c019ae3558d Mon Sep 17 00:00:00 2001 From: Daniel Holbach Date: Mon, 12 Apr 2021 10:57:51 +0200 Subject: [PATCH 23/25] Bounce kubernetes testing versions This update the test matrix to latest set of 3 minor k8s releases Fixes: #347 Co-Authored-By: Jean-Philippe Evrard --- .github/kind-cluster-1.19.yaml | 10 +++++----- .github/kind-cluster-1.20.yaml | 10 +++++----- .../{kind-cluster-1.18.yaml => kind-cluster-1.21.yaml} | 10 +++++----- .github/workflows/on-pr.yaml | 4 ++-- .github/workflows/periodics-daily.yaml | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) rename .github/{kind-cluster-1.18.yaml => kind-cluster-1.21.yaml} (50%) diff --git a/.github/kind-cluster-1.19.yaml b/.github/kind-cluster-1.19.yaml index 958c86d..05fedd2 100644 --- a/.github/kind-cluster-1.19.yaml +++ b/.github/kind-cluster-1.19.yaml @@ -2,12 +2,12 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.19.4 + image: kindest/node:v1.19.7 - role: control-plane - image: kindest/node:v1.19.4 + image: kindest/node:v1.19.7 - role: control-plane - image: kindest/node:v1.19.4 + image: kindest/node:v1.19.7 - role: worker - image: kindest/node:v1.19.4 + image: kindest/node:v1.19.7 - role: worker - image: kindest/node:v1.19.4 + image: kindest/node:v1.19.7 diff --git a/.github/kind-cluster-1.20.yaml b/.github/kind-cluster-1.20.yaml index fb9dd28..cf06c7f 100644 --- a/.github/kind-cluster-1.20.yaml +++ b/.github/kind-cluster-1.20.yaml @@ -2,12 +2,12 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: "kindest/node:v1.20.0" + image: "kindest/node:v1.20.2" - role: control-plane - image: "kindest/node:v1.20.0" + image: "kindest/node:v1.20.2" - role: control-plane - image: "kindest/node:v1.20.0" + image: "kindest/node:v1.20.2" - role: worker - image: "kindest/node:v1.20.0" + image: "kindest/node:v1.20.2" - role: worker - image: "kindest/node:v1.20.0" + image: "kindest/node:v1.20.2" diff --git a/.github/kind-cluster-1.18.yaml b/.github/kind-cluster-1.21.yaml similarity index 50% rename from .github/kind-cluster-1.18.yaml rename to .github/kind-cluster-1.21.yaml index 699f25a..c957bb7 100644 --- a/.github/kind-cluster-1.18.yaml +++ b/.github/kind-cluster-1.21.yaml @@ -2,12 +2,12 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.18.8 + image: kindest/node:v1.21.1 - role: control-plane - image: kindest/node:v1.18.8 + image: kindest/node:v1.21.1 - role: control-plane - image: kindest/node:v1.18.8 + image: kindest/node:v1.21.1 - role: worker - image: kindest/node:v1.18.8 + image: kindest/node:v1.21.1 - role: worker - image: kindest/node:v1.18.8 + image: kindest/node:v1.21.1 diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 1c9c8bb..f6758aa 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -97,9 +97,9 @@ jobs: fail-fast: false matrix: kubernetes: - - "1.18" - "1.19" - "1.20" + - "1.21" steps: - uses: actions/checkout@v2 - name: Find go version @@ -169,7 +169,7 @@ jobs: fail-fast: false matrix: kubernetes: - - "1.20" + - "1.21" steps: - uses: actions/checkout@v2 - name: Find go version diff --git a/.github/workflows/periodics-daily.yaml b/.github/workflows/periodics-daily.yaml index 6e5daa1..8e9e345 100644 --- a/.github/workflows/periodics-daily.yaml +++ b/.github/workflows/periodics-daily.yaml @@ -74,9 +74,9 @@ jobs: strategy: matrix: kubernetes: - - "1.18" - "1.19" - "1.20" + - "1.21" steps: - uses: actions/checkout@v2 - name: Find go version @@ -132,4 +132,4 @@ jobs: env: DEBUG: true run: | - ./tests/kind/follow-coordinated-reboot.sh \ No newline at end of file + ./tests/kind/follow-coordinated-reboot.sh From 89d1fe497c022ced428db809c68d55ffd121f98c Mon Sep 17 00:00:00 2001 From: Daniel Holbach Date: Mon, 17 May 2021 13:54:49 +0200 Subject: [PATCH 24/25] use latest kind Signed-off-by: Daniel Holbach --- .github/workflows/on-pr-charts.yaml | 4 ++++ .github/workflows/on-pr.yaml | 3 +++ .github/workflows/periodics-daily.yaml | 1 + 3 files changed, 8 insertions(+) diff --git a/.github/workflows/on-pr-charts.yaml b/.github/workflows/on-pr-charts.yaml index 011bbe7..c9a742b 100644 --- a/.github/workflows/on-pr-charts.yaml +++ b/.github/workflows/on-pr-charts.yaml @@ -36,6 +36,8 @@ jobs: - name: Create default kind cluster uses: helm/kind-action@v1.1.0 + with: + version: v0.11.0 if: ${{ matrix.test-action == 'install' }} - name: Run chart tests @@ -52,6 +54,8 @@ jobs: # Default name for helm/kind-action kind clusters is "chart-testing" - name: Create 1 node kind cluster uses: helm/kind-action@v1.1.0 + with: + version: v0.11.0 - name: Deploy kured on default namespace with its helm chart run: | diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index f6758aa..a901a30 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -130,6 +130,7 @@ jobs: uses: helm/kind-action@v1.1.0 with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml + version: v0.11.0 - name: Preload previously built images onto kind cluster run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing @@ -198,6 +199,8 @@ jobs: # Default name for helm/kind-action kind clusters is "chart-testing" - name: Create 1 node kind cluster uses: helm/kind-action@v1.1.0 + with: + version: v0.11.0 - name: Preload previously built images onto kind cluster run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing diff --git a/.github/workflows/periodics-daily.yaml b/.github/workflows/periodics-daily.yaml index 8e9e345..3636d2f 100644 --- a/.github/workflows/periodics-daily.yaml +++ b/.github/workflows/periodics-daily.yaml @@ -103,6 +103,7 @@ jobs: uses: helm/kind-action@v1.1.0 with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml + version: v0.11.0 - name: Deploy kured on default namespace with its helm chart run: | From 0cbc2d58d2e6cf4d327518523b95b558fa7a317d Mon Sep 17 00:00:00 2001 From: Christian Kotzbauer Date: Wed, 19 May 2021 15:17:02 +0200 Subject: [PATCH 25/25] doc: add compat-line for 1.7.0 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1d91c16..ad85b15 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ server: | kured | kubectl | k8s.io/client-go | k8s.io/apimachinery | expected kubernetes compatibility | |-------|---------|------------------|---------------------|-----------------------------------| | main | 1.20.5 | v0.20.5 | v0.20.5 | 1.19.x, 1.20.x, 1.21.x | +| 1.7.0 | 1.20.5 | v0.20.5 | v0.20.5 | 1.19.x, 1.20.x, 1.21.x | | 1.6.1 | 1.19.4 | v0.19.4 | v0.19.4 | 1.18.x, 1.19.x, 1.20.x | | 1.5.1 | 1.18.8 | v0.18.8 | v0.18.8 | 1.17.x, 1.18.x, 1.19.x | | 1.4.4 | 1.17.7 | v0.17.0 | v0.17.0 | 1.16.x, 1.17.x, 1.18.x |