From 8aafc8905e3f36df0ea92ff5d219c7e0943f9ad3 Mon Sep 17 00:00:00 2001 From: Adam Harrison Date: Thu, 13 Apr 2017 16:59:57 +0100 Subject: [PATCH] Initial implementation --- .gitignore | 4 + Gopkg.lock | 235 +++++++++++++++++++++++++++ Gopkg.toml | 20 +++ LICENSE | 191 ++++++++++++++++++++++ Makefile | 35 +++++ README.md | 76 +++++++++ cmd/kured/Dockerfile | 5 + cmd/kured/main.go | 245 +++++++++++++++++++++++++++++ cmd/kured/pflags.go | 31 ++++ cmd/prom-active-alerts/main.go | 23 +++ kured-ds.yaml | 45 ++++++ pkg/alerts/prometheus.go | 42 +++++ pkg/daemonsetlock/daemonsetlock.go | 122 ++++++++++++++ pkg/delaytick/delaytick.go | 22 +++ 14 files changed, 1096 insertions(+) create mode 100644 .gitignore create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 cmd/kured/Dockerfile create mode 100644 cmd/kured/main.go create mode 100644 cmd/kured/pflags.go create mode 100644 cmd/prom-active-alerts/main.go create mode 100644 kured-ds.yaml create mode 100644 pkg/alerts/prometheus.go create mode 100644 pkg/daemonsetlock/daemonsetlock.go create mode 100644 pkg/delaytick/delaytick.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..400ca4d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +cmd/kured/kured +cmd/prom-active-alerts/prom-active-alerts +vendor +build diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..d43c8ed --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,235 @@ +memo = "758e136fb047a3fbb8746d2e83a9e11d341db8e16e4ddc78bc0e8b41038f023b" + +[[projects]] + name = "github.com/PuerkitoBio/purell" + packages = ["."] + revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/PuerkitoBio/urlesc" + packages = ["."] + revision = "bbf7a2afc14f93e1e0a5c06df524fbd75e5031e5" + +[[projects]] + branch = "master" + name = "github.com/Sirupsen/logrus" + packages = ["."] + revision = "abc6f20dabf4b10195f233ad21ea6c5ba33acae0" + +[[projects]] + branch = "master" + name = "github.com/blang/semver" + packages = ["."] + revision = "60ec3488bfea7cca02b021d106d9911120d25fe9" + +[[projects]] + branch = "master" + name = "github.com/cloudfoundry-incubator/candiedyaml" + packages = ["."] + revision = "99c3df83b51532e3615f851d8c2dbb638f5313bf" + +[[projects]] + branch = "master" + name = "github.com/coreos/go-oidc" + packages = ["http","jose","key","oauth2","oidc"] + revision = "48c41f8e5a608ae49cbff1d977dd060815a8bb9f" + +[[projects]] + branch = "master" + name = "github.com/coreos/go-systemd" + packages = ["journal"] + revision = "fa8411dcbcbad22b8542b0433914ef68b123f989" + +[[projects]] + branch = "master" + name = "github.com/coreos/pkg" + packages = ["capnslog","health","httputil","timeutil"] + revision = "3ac0863d7acf3bc44daf49afef8919af12f704ef" + +[[projects]] + branch = "master" + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "5215b55f46b2b919f50a1df0eaa5886afe4e3b3d" + +[[projects]] + branch = "master" + name = "github.com/docker/distribution" + packages = ["digest","reference"] + revision = "7365003236ca58bd7fa17ef1459328d13301d7d5" + +[[projects]] + branch = "master" + name = "github.com/emicklei/go-restful" + packages = [".","log","swagger"] + revision = "b14c3a95fc27c52959d2eddc85066da3c14bf269" + +[[projects]] + branch = "master" + name = "github.com/ghodss/yaml" + packages = ["."] + revision = "aa0c862057666179de291b67d9f093d12b5a8473" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/jsonpointer" + packages = ["."] + revision = "779f45308c19820f1a69e9a4cd965f496e0da10f" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/jsonreference" + packages = ["."] + revision = "36d33bfe519efae5632669801b180bf1a245da3b" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/spec" + packages = ["."] + revision = "e51c28f07047ad90caff03f6450908720d337e0c" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/swag" + packages = ["."] + revision = "24ebf76d720bab64f62824d76bced3184a65490d" + +[[projects]] + branch = "master" + name = "github.com/gogo/protobuf" + packages = ["proto","sortkeys"] + revision = "e33835a643a970c11ac74f6333f5f6866387a101" + +[[projects]] + branch = "master" + name = "github.com/golang/glog" + packages = ["."] + revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" + +[[projects]] + branch = "master" + name = "github.com/golang/protobuf" + packages = ["proto"] + revision = "2bba0603135d7d7f5cb73b2125beeda19c09f4ef" + +[[projects]] + branch = "master" + name = "github.com/google/gofuzz" + packages = ["."] + revision = "fd52762d25a41827db7ef64c43756fd4b9f7e382" + +[[projects]] + branch = "master" + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + +[[projects]] + branch = "master" + name = "github.com/jonboulle/clockwork" + packages = ["."] + revision = "2eee05ed794112d45db504eb05aa693efd2b8b09" + +[[projects]] + branch = "master" + name = "github.com/juju/ratelimit" + packages = ["."] + revision = "77ed1c8a01217656d2080ad51981f6e99adaa177" + +[[projects]] + branch = "master" + name = "github.com/mailru/easyjson" + packages = ["buffer","jlexer","jwriter"] + revision = "2af9a745a611440bab0528e5ac19b2805a1c50eb" + +[[projects]] + branch = "master" + name = "github.com/pborman/uuid" + packages = ["."] + revision = "c55201b036063326c5b1b89ccfe45a184973d073" + +[[projects]] + branch = "master" + name = "github.com/prometheus/client_golang" + packages = ["api/prometheus"] + revision = "5636dc67ae776adf5590da7349e70fbb9559972d" + +[[projects]] + branch = "master" + name = "github.com/prometheus/common" + packages = ["model"] + revision = "ebdfc6da46522d58825777cf1f90490a5b1ef1d8" + +[[projects]] + branch = "master" + name = "github.com/spf13/cobra" + packages = ["."] + revision = "b24564e919247d7c870fe0ed3738c98d8741aca4" + +[[projects]] + branch = "master" + name = "github.com/spf13/pflag" + packages = ["."] + revision = "367864438f1b1a3c7db4da06a2f55b144e6784e0" + +[[projects]] + branch = "master" + name = "github.com/ugorji/go" + packages = ["codec"] + revision = "3487a5545b3d480987dfb0492035299077fab33a" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = ["context","context/ctxhttp","http2","http2/hpack","idna"] + revision = "2a35e686583654a1b89ca79c4ac78cb3d6529ca3" + +[[projects]] + branch = "master" + name = "golang.org/x/oauth2" + packages = [".","google","internal","jws","jwt"] + revision = "e86e2718db89775a4604abc10a5d3a5672e7336e" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["unix"] + revision = "a646d33e2ee3172a661fc09bca23bb4889a41bc8" + +[[projects]] + branch = "master" + name = "golang.org/x/text" + packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm","width"] + revision = "a9a820217f98f7c8a207ec1e45a874e1fe12c478" + +[[projects]] + name = "google.golang.org/appengine" + packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"] + revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "google.golang.org/cloud" + packages = ["compute/metadata","internal"] + revision = "d96a09b3489f3183f15b676a5f8d04af06bdeb0e" + +[[projects]] + branch = "master" + name = "gopkg.in/inf.v0" + packages = ["."] + revision = "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4" + +[[projects]] + branch = "v2" + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b" + +[[projects]] + name = "k8s.io/client-go" + packages = ["discovery","kubernetes","kubernetes/typed/apps/v1beta1","kubernetes/typed/authentication/v1beta1","kubernetes/typed/authorization/v1beta1","kubernetes/typed/autoscaling/v1","kubernetes/typed/batch/v1","kubernetes/typed/batch/v2alpha1","kubernetes/typed/certificates/v1alpha1","kubernetes/typed/core/v1","kubernetes/typed/extensions/v1beta1","kubernetes/typed/policy/v1beta1","kubernetes/typed/rbac/v1alpha1","kubernetes/typed/storage/v1beta1","pkg/api","pkg/api/errors","pkg/api/install","pkg/api/meta","pkg/api/meta/metatypes","pkg/api/resource","pkg/api/unversioned","pkg/api/v1","pkg/api/validation/path","pkg/apimachinery","pkg/apimachinery/announced","pkg/apimachinery/registered","pkg/apis/apps","pkg/apis/apps/install","pkg/apis/apps/v1beta1","pkg/apis/authentication","pkg/apis/authentication/install","pkg/apis/authentication/v1beta1","pkg/apis/authorization","pkg/apis/authorization/install","pkg/apis/authorization/v1beta1","pkg/apis/autoscaling","pkg/apis/autoscaling/install","pkg/apis/autoscaling/v1","pkg/apis/batch","pkg/apis/batch/install","pkg/apis/batch/v1","pkg/apis/batch/v2alpha1","pkg/apis/certificates","pkg/apis/certificates/install","pkg/apis/certificates/v1alpha1","pkg/apis/extensions","pkg/apis/extensions/install","pkg/apis/extensions/v1beta1","pkg/apis/policy","pkg/apis/policy/install","pkg/apis/policy/v1beta1","pkg/apis/rbac","pkg/apis/rbac/install","pkg/apis/rbac/v1alpha1","pkg/apis/storage","pkg/apis/storage/install","pkg/apis/storage/v1beta1","pkg/auth/user","pkg/conversion","pkg/conversion/queryparams","pkg/fields","pkg/genericapiserver/openapi/common","pkg/labels","pkg/runtime","pkg/runtime/serializer","pkg/runtime/serializer/json","pkg/runtime/serializer/protobuf","pkg/runtime/serializer/recognizer","pkg/runtime/serializer/streaming","pkg/runtime/serializer/versioning","pkg/selection","pkg/third_party/forked/golang/reflect","pkg/third_party/forked/golang/template","pkg/types","pkg/util","pkg/util/cert","pkg/util/clock","pkg/util/errors","pkg/util/flowcontrol","pkg/util/framer","pkg/util/integer","pkg/util/intstr","pkg/util/json","pkg/util/jsonpath","pkg/util/labels","pkg/util/net","pkg/util/parsers","pkg/util/rand","pkg/util/runtime","pkg/util/sets","pkg/util/uuid","pkg/util/validation","pkg/util/validation/field","pkg/util/wait","pkg/util/yaml","pkg/version","pkg/watch","pkg/watch/versioned","plugin/pkg/client/auth","plugin/pkg/client/auth/gcp","plugin/pkg/client/auth/oidc","rest","tools/clientcmd/api","tools/metrics","transport"] + revision = "e121606b0d09b2e1c467183ee46217fa85a6b672" + version = "v2.0.0" diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..6fbe489 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,20 @@ + +[[dependencies]] + branch = "master" + name = "github.com/Sirupsen/logrus" + +[[dependencies]] + branch = "master" + name = "github.com/prometheus/client_golang" + +[[dependencies]] + branch = "master" + name = "github.com/prometheus/common" + +[[dependencies]] + branch = "master" + name = "github.com/spf13/cobra" + +[[dependencies]] + name = "k8s.io/client-go" + version = "^2.0.0" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..64a35d0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2017 Weaveworks Ltd. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..eb8e5e9 --- /dev/null +++ b/Makefile @@ -0,0 +1,35 @@ +.DEFAULT: all +.PHONY: all clean image publish-image minikube-publish + +DH_ORG=weaveworks +IMAGE_TAG=latest + +all: image + +clean: + go clean + rm -f cmd/kured/kured + rm -rf ./build + +godeps=$(shell go get $1 && go list -f '{{join .Deps "\n"}}' $1 | grep -v /vendor/ | xargs go list -f '{{if not .Standard}}{{ $$dep := . }}{{range .GoFiles}}{{$$dep.Dir}}/{{.}} {{end}}{{end}}') + +DEPS=$(call godeps,./cmd/kured) +VERSION=git-$(shell git rev-parse --short=12 HEAD) + +cmd/kured/kured: $(DEPS) +cmd/kured/kured: cmd/kured/*.go + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X main.version=$(VERSION)" -o $@ cmd/kured/*.go + +build/.image.done: cmd/kured/Dockerfile cmd/kured/kured + mkdir -p build + cp $^ build + sudo -E docker build -t $(DH_ORG)/kured:$(IMAGE_TAG) -f build/Dockerfile ./build + touch $@ + +image: build/.image.done + +publish-image: image + sudo -E docker push $(DH_ORG)/kured:$(IMAGE_TAG) + +minikube-publish: image + sudo -E docker save $(DH_ORG)/kured:$(IMAGE_TAG) | (eval $$(minikube docker-env) && docker load) diff --git a/README.md b/README.md new file mode 100644 index 0000000..1f42f38 --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ + +* [Introduction](#introduction) +* [Configuration](#configuration) + * [Reboot Sentinel File & Period](#reboot-sentinel-file-&-period) + * [Blocking Reboots via Alerts](#blocking-reboots-via-alerts) + * [Overriding Lock Configuration](#overriding-lock-configuration) +* [Building](#building) + +## Introduction + +Kured (KUbernetes REboot Daemon) is a Kubernetes daemonset that +performs safe automatic node reboots when it is requested by the +package management system of the underlying OS. + +* Watches for the presence of a reboot sentinel e.g. `/var/run/reboot-required` +* Utilises a lock in the API server to ensure only one node reboots at + a time +* Optionally defers reboots in the presence of active Prometheus alerts +* Cordons & drains worker nodes before reboot, uncordoning them after + +## Configuration + +The following arguments can be passed to kured via the daemonset pod template: + +``` +Flags: + --alert-filter-regexp value alert names to ignore when checking for active alerts + --ds-name string namespace containing daemonset on which to place lock (default "kube-system") + --ds-namespace string name of daemonset on which to place lock (default "kured") + --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") + --period int reboot check period in minutes (default 60) + --prometheus-url string Prometheus instance to probe for active alerts + --reboot-sentinel string path to file whose existence signals need to reboot (default "/var/run/reboot-required") +``` + +### Reboot Sentinel File & Period + +By default kured checks for the existence of +`/var/run/reboot-required` every sixty minutes; you can override these +values with `--reboot-sentinel` and `--period`. Each instance of the +reboot uses a random offset derived from the period on startup so that +nodes don't all contend for the lock simultaneously. + +### Blocking Reboots via Alerts + +You may find it desirable to block automatic node reboots when there +are active alerts - you can do so by providing the URL of your +Prometheus server: + +``` +--prometheus-url=http://prometheus.monitoring.svc.cluster.local +``` + +By default the presence of *any* active (pending or firing) alerts +will block reboots, however you can ignore specific alerts: + +``` +--alert-filter-regexp=^(BenignAlert|AnotherBenignAlert|...$ +``` + +### Overriding Lock Configuration + +The `--ds-name` and `--ds-namespace` arguments should match the name and +namespace of the daemonset used to deploy the reboot daemon - the locking is +implemented by means of an annotation on this resource. The defaults match +the daemonset YAML provided in the repository. + +Similarly `--lock-annotation` can be used to change the name of the +annotation kured will use to store the lock, but the default is almost +certainly safe. + +## Building + +``` +dep ensure && make +``` diff --git a/cmd/kured/Dockerfile b/cmd/kured/Dockerfile new file mode 100644 index 0000000..27c4340 --- /dev/null +++ b/cmd/kured/Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu +ADD https://storage.googleapis.com/kubernetes-release/release/v1.4.8/bin/linux/amd64/kubectl /usr/bin/kubectl +RUN chmod 0755 /usr/bin/kubectl +COPY ./kured /usr/bin/kured +ENTRYPOINT ["/usr/bin/kured"] diff --git a/cmd/kured/main.go b/cmd/kured/main.go new file mode 100644 index 0000000..b893f91 --- /dev/null +++ b/cmd/kured/main.go @@ -0,0 +1,245 @@ +package main + +import ( + "math/rand" + "os" + "os/exec" + "regexp" + "time" + + log "github.com/Sirupsen/logrus" + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/pkg/api/v1" + "k8s.io/client-go/rest" + + "github.com/weaveworks/kured/pkg/alerts" + "github.com/weaveworks/kured/pkg/daemonsetlock" + "github.com/weaveworks/kured/pkg/delaytick" +) + +var ( + version = "unreleased" + period int + dsNamespace string + dsName string + lockAnnotation string + prometheusURL string + alertFilter *regexp.Regexp + rebootSentinel string +) + +func main() { + rootCmd := &cobra.Command{ + Use: "kured", + Short: "Kubernetes Reboot Daemon", + Run: root} + + rootCmd.PersistentFlags().IntVar(&period, "period", 60, + "reboot check period in minutes") + rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-name", "kube-system", + "namespace containing daemonset on which to place lock") + rootCmd.PersistentFlags().StringVar(&dsName, "ds-namespace", "kured", + "name of daemonset on which to place lock") + rootCmd.PersistentFlags().StringVar(&lockAnnotation, "lock-annotation", "weave.works/kured-node-lock", + "annotation in which to record locking node") + rootCmd.PersistentFlags().StringVar(&prometheusURL, "prometheus-url", "", + "Prometheus instance to probe for active alerts") + rootCmd.PersistentFlags().Var(®expValue{&alertFilter}, "alert-filter-regexp", + "alert names to ignore when checking for active alerts") + rootCmd.PersistentFlags().StringVar(&rebootSentinel, "reboot-sentinel", "/var/run/reboot-required", + "path to file whose existence signals need to reboot") + + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) + } +} + +func rebootRequired() bool { + _, err := os.Stat(rebootSentinel) + switch { + case err == nil: + log.Infof("Reboot required") + return true + case os.IsNotExist(err): + log.Infof("Reboot not required") + return false + default: + log.Fatalf("Unable to determine if reboot required: %v", err) + return false // unreachable; prevents compilation error + } +} + +func rebootBlocked() bool { + if prometheusURL != "" { + count, err := alerts.PrometheusCountActive(prometheusURL, alertFilter) + if err != nil { + log.Warnf("Reboot blocked: prometheus query error: %v", err) + return true + } + if count > 0 { + log.Warnf("Reboot blocked: %d active alerts", count) + return true + } + } + return false +} + +func holding(lock *daemonsetlock.DaemonSetLock, metadata interface{}) bool { + holding, err := lock.Test(metadata) + if err != nil { + log.Fatalf("Error testing lock: %v", err) + } + if holding { + log.Infof("Holding lock") + } + return holding +} + +func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}) bool { + holding, holder, err := lock.Acquire(metadata) + switch { + case err != nil: + log.Fatalf("Error acquiring lock: %v", err) + return false + case !holding: + log.Warnf("Lock already held: %v", holder) + return false + default: + log.Infof("Acquired reboot lock") + return true + } +} + +func release(lock *daemonsetlock.DaemonSetLock) { + log.Infof("Releasing lock") + if err := lock.Release(); err != nil { + log.Fatalf("Error releasing lock: %v", err) + } +} + +func drain(nodeID string) { + log.Infof("Draining node %s", nodeID) + drainCmd := exec.Command("/usr/bin/kubectl", "drain", + "--ignore-daemonsets", "--delete-local-data", "--force", nodeID) + if err := drainCmd.Run(); err != nil { + log.Fatalf("Error invoking drain command: %v", err) + } +} + +func uncordon(nodeID string) { + log.Infof("Uncordoning node %s", nodeID) + uncordonCmd := exec.Command("/usr/bin/kubectl", "uncordon", nodeID) + if err := uncordonCmd.Run(); err != nil { + log.Fatalf("Error invoking uncordon command: %v", err) + } +} + +func waitForDrain(client *kubernetes.Clientset, nodeID string) { + for { + var unterminated int + + namespaces, err := client.CoreV1().Namespaces().List(v1.ListOptions{}) + if err != nil { + log.Fatalf("Error waiting for drain: %v", err) + } + + for _, namespace := range namespaces.Items { + drainCandidates := v1.ListOptions{LabelSelector: "ignore_on_drain!=true"} + pods, err := client.CoreV1().Pods(namespace.ObjectMeta.Name).List(drainCandidates) + if err != nil { + log.Fatalf("Error waiting for drain: %v", err) + } + + for _, pod := range pods.Items { + if pod.Spec.NodeName == nodeID && + pod.Status.Phase != "Succeeded" && + pod.Status.Phase != "Failed" { + unterminated++ + } + } + } + + if unterminated == 0 { + return + } + + log.Infof("Waiting for %d pods to terminate", unterminated) + time.Sleep(time.Minute) + } +} + +func reboot() { + log.Infof("Commanding reboot") + // Relies on /var/run/dbus/system_bus_socket bind mount to talk to systemd + rebootCmd := exec.Command("/bin/systemctl", "reboot") + if err := rebootCmd.Run(); err != nil { + log.Fatalf("Error invoking reboot command: %v", err) + } +} + +func waitForReboot() { + for { + log.Infof("Waiting for reboot") + time.Sleep(time.Minute) + } +} + +// nodeMeta is used to remember information across reboots +type nodeMeta struct { + Unschedulable bool `json:"unschedulable"` +} + +func root(cmd *cobra.Command, args []string) { + log.Infof("Kubernetes Reboot Daemon: %s", version) + + nodeID := os.Getenv("KURED_NODE_ID") + if nodeID == "" { + log.Fatal("KURED_NODE_ID environment variable required") + } + + log.Infof("Node ID: %s", nodeID) + log.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation) + log.Infof("Reboot Sentinel: %s every %d minutes", rebootSentinel, period) + + config, err := rest.InClusterConfig() + if err != nil { + log.Fatal(err) + } + + client, err := kubernetes.NewForConfig(config) + if err != nil { + log.Fatal(err) + } + + lock := daemonsetlock.New(client, nodeID, dsNamespace, dsName, lockAnnotation) + + nodeMeta := nodeMeta{} + if holding(lock, &nodeMeta) { + if !nodeMeta.Unschedulable { + uncordon(nodeID) + } + release(lock) + } else { + node, err := client.CoreV1().Nodes().Get(nodeID) + if err != nil { + log.Fatal(err) + } + nodeMeta.Unschedulable = node.Spec.Unschedulable + } + + source := rand.NewSource(time.Now().UnixNano()) + tick := delaytick.New(source, time.Minute*time.Duration(period)) + for _ = range tick { + if rebootRequired() && !rebootBlocked() && acquire(lock, &nodeMeta) { + if !nodeMeta.Unschedulable { + drain(nodeID) + waitForDrain(client, nodeID) + } + reboot() + break + } + } + + waitForReboot() +} diff --git a/cmd/kured/pflags.go b/cmd/kured/pflags.go new file mode 100644 index 0000000..b6eb5ab --- /dev/null +++ b/cmd/kured/pflags.go @@ -0,0 +1,31 @@ +package main + +import ( + "regexp" +) + +type regexpValue struct { + value **regexp.Regexp +} + +func (rev *regexpValue) String() string { + if *rev.value == nil { + return "" + } + return (*rev.value).String() +} + +func (rev *regexpValue) Set(s string) error { + value, err := regexp.Compile(s) + if err != nil { + return err + } + + *rev.value = value + + return nil +} + +func (rev *regexpValue) Type() string { + return "regexp.Regexp" +} diff --git a/cmd/prom-active-alerts/main.go b/cmd/prom-active-alerts/main.go new file mode 100644 index 0000000..a2dec5c --- /dev/null +++ b/cmd/prom-active-alerts/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + "log" + "os" + "regexp" + + "github.com/weaveworks/kured/pkg/alerts" +) + +func main() { + if len(os.Args) != 3 { + log.Fatalf("USAGE: %s ", os.Args[0]) + } + + count, err := alerts.PrometheusCountActive(os.Args[1], regexp.MustCompile(os.Args[2])) + if err != nil { + log.Fatal(err) + } + + fmt.Println(count) +} diff --git a/kured-ds.yaml b/kured-ds.yaml new file mode 100644 index 0000000..a1ef249 --- /dev/null +++ b/kured-ds.yaml @@ -0,0 +1,45 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kured + namespace: kube-system +spec: + template: + metadata: + labels: + name: kured + spec: + containers: + - name: kured + image: weaveworks/kured + imagePullPolicy: IfNotPresent + command: + - /usr/bin/kured +# args: +# - --alert-filter-regexp=^(BenignAlert|AnotherBenignAlert)$ +# - --ds-name=kube-system +# - --ds-namespace=kured +# - --lock-annotation=weave.works/kured-node-lock +# - --period=60 +# - --prometheus-url=http://prometheus.monitoring.svc.cluster.local +# - --reboot-sentinel=/var/run/reboot-required + env: + # Pass in the name of the node on which this pod is scheduled + # for use with drain/uncordon operations and lock acquisition + - name: KURED_NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: true + volumeMounts: + # Needed for two purposes: + # * Testing for the existence of /var/run/reboot-required + # * Accessing /var/run/dbus/system_bus_socket to effect reboot + - name: hostrun + mountPath: /var/run + restartPolicy: Always + volumes: + - name: hostrun + hostPath: + path: /var/run diff --git a/pkg/alerts/prometheus.go b/pkg/alerts/prometheus.go new file mode 100644 index 0000000..39455df --- /dev/null +++ b/pkg/alerts/prometheus.go @@ -0,0 +1,42 @@ +package alerts + +import ( + "context" + "fmt" + "regexp" + "time" + + "github.com/prometheus/client_golang/api/prometheus" + "github.com/prometheus/common/model" +) + +// Return true if there are any active (e.g. pending or firing) alerts +func PrometheusCountActive(prometheusURL string, filter *regexp.Regexp) (int, error) { + client, err := prometheus.New(prometheus.Config{Address: prometheusURL}) + if err != nil { + return 0, err + } + + queryAPI := prometheus.NewQueryAPI(client) + + value, err := queryAPI.Query(context.Background(), "ALERTS", time.Now()) + if err != nil { + return 0, err + } + + if value.Type() == model.ValVector { + if vector, ok := value.(model.Vector); ok { + var count int + for _, sample := range vector { + if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert { + if filter == nil || !filter.MatchString(string(alertName)) { + count++ + } + } + } + return count, nil + } + } + + return 0, fmt.Errorf("Unexpected value type: %v", value) +} diff --git a/pkg/daemonsetlock/daemonsetlock.go b/pkg/daemonsetlock/daemonsetlock.go new file mode 100644 index 0000000..0cc71d7 --- /dev/null +++ b/pkg/daemonsetlock/daemonsetlock.go @@ -0,0 +1,122 @@ +package daemonsetlock + +import ( + "encoding/json" + "fmt" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/pkg/api/errors" + "k8s.io/client-go/pkg/api/unversioned" +) + +type DaemonSetLock struct { + client *kubernetes.Clientset + nodeID string + namespace string + name string + annotation string +} + +type lockAnnotationValue struct { + NodeID string `json:"nodeID"` + Metadata interface{} `json:"metadata,omitempty"` +} + +func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation string) *DaemonSetLock { + return &DaemonSetLock{client, nodeID, namespace, name, annotation} +} + +func (dsl *DaemonSetLock) Acquire(metadata interface{}) (acquired bool, owner string, err error) { + for { + ds, err := dsl.client.ExtensionsV1beta1().DaemonSets(dsl.namespace).Get(dsl.name) + if err != nil { + return false, "", err + } + + valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] + if exists { + value := lockAnnotationValue{} + if err := json.Unmarshal([]byte(valueString), &value); err != nil { + return false, "", err + } + return value.NodeID == dsl.nodeID, value.NodeID, nil + } + + if ds.ObjectMeta.Annotations == nil { + ds.ObjectMeta.Annotations = make(map[string]string) + } + value := lockAnnotationValue{NodeID: dsl.nodeID, Metadata: metadata} + valueBytes, err := json.Marshal(&value) + if err != nil { + return false, "", err + } + ds.ObjectMeta.Annotations[dsl.annotation] = string(valueBytes) + + _, err = dsl.client.ExtensionsV1beta1().DaemonSets(dsl.namespace).Update(ds) + if err != nil { + if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == unversioned.StatusReasonConflict { + // Something else updated the resource between us reading and writing - try again soon + time.Sleep(time.Second) + continue + } else { + return false, "", err + } + } + return true, dsl.nodeID, nil + } +} + +func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) { + ds, err := dsl.client.ExtensionsV1beta1().DaemonSets(dsl.namespace).Get(dsl.name) + if err != nil { + return false, err + } + + valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] + if exists { + value := lockAnnotationValue{Metadata: metadata} + if err := json.Unmarshal([]byte(valueString), &value); err != nil { + return false, err + } + return value.NodeID == dsl.nodeID, nil + } + + return false, nil +} + +func (dsl *DaemonSetLock) Release() error { + for { + ds, err := dsl.client.ExtensionsV1beta1().DaemonSets(dsl.namespace).Get(dsl.name) + if err != nil { + return err + } + + valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] + if exists { + value := lockAnnotationValue{} + if err := json.Unmarshal([]byte(valueString), &value); err != nil { + return err + } + if value.NodeID != dsl.nodeID { + return fmt.Errorf("Not lock holder: %v", value.NodeID) + } + } else { + return fmt.Errorf("Lock not held") + } + + delete(ds.ObjectMeta.Annotations, dsl.annotation) + + _, err = dsl.client.ExtensionsV1beta1().DaemonSets(dsl.namespace).Update(ds) + if err != nil { + if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == unversioned.StatusReasonConflict { + // Something else updated the resource between us reading and writing - try again soon + time.Sleep(time.Second) + continue + } else { + return err + } + } + return nil + } +} diff --git a/pkg/delaytick/delaytick.go b/pkg/delaytick/delaytick.go new file mode 100644 index 0000000..4673bb9 --- /dev/null +++ b/pkg/delaytick/delaytick.go @@ -0,0 +1,22 @@ +package delaytick + +import ( + "math/rand" + "time" +) + +// Tick regularly after an initial delay randomly distributed between d/2 and d + d/2 +func New(s rand.Source, d time.Duration) <-chan time.Time { + c := make(chan time.Time) + + go func() { + random := rand.New(s) + time.Sleep(time.Duration(float64(d)/2 + float64(d)*random.Float64())) + c <- time.Now() + for t := range time.Tick(d) { + c <- t + } + }() + + return c +}