Merge branch 'main' into master

2026-08-23 21:36:33 +00:00 · 2021-05-19 18:49:37 +02:00
parent 2850417e48 4f8e9a0761
commit 0bd22c7c56
18 changed files with 578 additions and 207 deletions
@@ -2,12 +2,12 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
-  image: kindest/node:v1.19.4
+  image: kindest/node:v1.19.7
 - role: control-plane
-  image: kindest/node:v1.19.4
+  image: kindest/node:v1.19.7
 - role: control-plane
-  image: kindest/node:v1.19.4
+  image: kindest/node:v1.19.7
 - role: worker
-  image: kindest/node:v1.19.4
+  image: kindest/node:v1.19.7
 - role: worker
-  image: kindest/node:v1.19.4
+  image: kindest/node:v1.19.7
@@ -2,12 +2,12 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
-  image: "kindest/node:v1.20.0"
+  image: "kindest/node:v1.20.2"
 - role: control-plane
-  image: "kindest/node:v1.20.0"
+  image: "kindest/node:v1.20.2"
 - role: control-plane
-  image: "kindest/node:v1.20.0"
+  image: "kindest/node:v1.20.2"
 - role: worker
-  image: "kindest/node:v1.20.0"
+  image: "kindest/node:v1.20.2"
 - role: worker
-  image: "kindest/node:v1.20.0"
+  image: "kindest/node:v1.20.2"
@@ -2,12 +2,12 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
-  image: kindest/node:v1.18.8
+  image: kindest/node:v1.21.1
 - role: control-plane
-  image: kindest/node:v1.18.8
+  image: kindest/node:v1.21.1
 - role: control-plane
-  image: kindest/node:v1.18.8
+  image: kindest/node:v1.21.1
 - role: worker
-  image: kindest/node:v1.18.8
+  image: kindest/node:v1.21.1
 - role: worker
-  image: kindest/node:v1.18.8
+  image: kindest/node:v1.21.1
@@ -1,5 +1,6 @@
 #This is just extra testing, for lint check, and basic installation
-#If those fail, no need to test the rest of the PR (github will cancel the rest of the builds)
+#Those can fail earlier than functional tests (shorter tests)
+# and give developer feedback soon if they didn't test themselves
 name: PR - charts
 on:
  pull_request:
@@ -11,7 +12,7 @@ jobs:
  # tackling that for us.
  # Fail-fast ensures that if one of those matrix job fail, the other one gets cancelled.
  test-chart:
-    name: Test helm chart
+    name: Test helm chart changes
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
@@ -35,7 +36,43 @@ jobs:

      - name: Create default kind cluster
        uses: helm/kind-action@v1.1.0
+        with:
+          version: v0.11.0
        if: ${{ matrix.test-action == 'install' }}

      - name: Run chart tests
        run: ct ${{ matrix.test-action }} --config .github/ct.yaml
+
+  # This doesn't re-use the ct actions, due to many limitations (auto tear down, no real testing)
+  deploy-chart:
+    name: Functional test of helm chart in its current state (needs published image of the helm chart)
+    runs-on: ubuntu-latest
+    needs: test-chart
+    steps:
+      - uses: actions/checkout@v2
+      
+      # Default name for helm/kind-action kind clusters is "chart-testing"
+      - name: Create 1 node kind cluster
+        uses: helm/kind-action@v1.1.0
+        with:
+          version: v0.11.0
+
+      - name: Deploy kured on default namespace with its helm chart
+        run: |
+          # Documented in official helm doc to live on the edge
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          # Refresh bins
+          hash -r
+          helm install kured ./charts/kured/ --set configuration.period=1m --wait
+          kubectl config set-context kind-chart-testing
+          kubectl get ds --all-namespaces
+          kubectl describe ds kured
+
+      - name: Test if successful deploy
+        uses: nick-invision/retry@v2.4.0
+        with:
+          timeout_minutes: 10
+          max_attempts: 10
+          retry_wait_seconds: 10
+          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
+          command: "kubectl get ds kured | grep -E 'kured.*1.*1.*1.*1.*1'"
@@ -84,17 +84,93 @@ jobs:
        with:
          image-name: docker.io/${{ github.repository_owner }}/kured:${{ github.sha }}

-  # If the PRs don't break the behaviour in the helm chart, we can simply publish the helm charts at the time of a branch commit.
-  e2e-helm:
-    name: "Functional test of helm chart, e2e testing"
+  # This ensures the latest code works with the manifests built from tree.
+  # It is useful for two things:
+  # - Test manifests changes (obviously), ensuring they don't break existing clusters
+  # - Ensure manifests work with the latest versions even with no manifest change
+  #     (compared to helm charts, manifests cannot easily template changes based on versions)
+  # Helm charts are _trailing_ releases, while manifests are done during development.
+  e2e-manifests:
+    name: End-to-End test with kured with code and manifests from HEAD
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        kubernetes:
+          - "1.19"
+          - "1.20"
+          - "1.21"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Find go version
+        run: |
+          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
+          echo "::set-output name=version::${GO_VERSION}"
+        id: awk_gomod
+      - name: Ensure go version
+        uses: actions/setup-go@v2
+        with:
+          go-version: "${{ steps.awk_gomod.outputs.version }}"
+      - name: Build artifacts
+        run: |
+          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
+          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" manifest
+
+      - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions
+        run: |
+          sudo bash << EOF
+              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
+              echo '{}' > /etc/docker/daemon.json
+              systemctl restart docker || journalctl --no-pager -n 500
+              systemctl status docker
+          EOF
+
+      # Default name for helm/kind-action kind clusters is "chart-testing"
+      - name: Create kind cluster with 5 nodes
+        uses: helm/kind-action@v1.1.0
+        with:
+          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
+          version: v0.11.0
+
+      - name: Preload previously built images onto kind cluster
+        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
+
+      - name: Do not wait for an hour before detecting the rebootSentinel
+        run: |
+          sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds.yaml
+
+      - name: Install kured with kubectl
+        run: |
+          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml
+
+      - name: Ensure kured is ready
+        uses: nick-invision/retry@v2.4.0
+        with:
+          timeout_minutes: 10
+          max_attempts: 10
+          retry_wait_seconds: 60
+          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
+          command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
+
+      - name: Create reboot sentinel files
+        run: |
+          ./tests/kind/create-reboot-sentinels.sh
+
+      - name: Follow reboot until success
+        env:
+          DEBUG: true
+        run: |
+          ./tests/kind/follow-coordinated-reboot.sh
+
+  scenario-prom-helm:
+    name: Test prometheus with latest code from HEAD (=overrides image of the helm chart)
    runs-on: ubuntu-latest
    # only build with oldest and newest supported, it should be good enough.
    strategy:
      fail-fast: false
      matrix:
        kubernetes:
-          - "1.18"
-          - "1.20"
+          - "1.21"
    steps:
      - uses: actions/checkout@v2
      - name: Find go version
@@ -111,7 +187,7 @@ jobs:
          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart

-      - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions"
+      - name: Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions
        run: |
          sudo bash << EOF
              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
@@ -121,10 +197,10 @@ jobs:
          EOF

      # Default name for helm/kind-action kind clusters is "chart-testing"
-      - name: Create 5 node kind cluster
-        uses: helm/kind-action@master
+      - name: Create 1 node kind cluster
+        uses: helm/kind-action@v1.1.0
        with:
-          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
+          version: v0.11.0

      - name: Preload previously built images onto kind cluster
        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
@@ -135,7 +211,7 @@ jobs:
          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
          # Refresh bins
          hash -r
-          helm install kured ./charts/kured/ --set configuration.period=1m
+          helm install kured ./charts/kured/ --wait --values ./charts/kured/ci/prometheus-values.yaml
          kubectl config set-context kind-chart-testing
          kubectl get ds --all-namespaces
          kubectl describe ds kured
@@ -146,68 +222,106 @@ jobs:
          timeout_minutes: 10
          max_attempts: 10
          retry_wait_seconds: 60
-          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = 5
-          command: "kubectl get ds kured | grep -E 'kured.*5.*5.*5.*5.*5' "
+          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE
+          command: "kubectl get ds kured | grep -E 'kured.*1.*1.*1.*1.*1' "
+
+      - name: Get metrics (healthy)
+        uses: nick-invision/retry@v2.4.0
+        with:
+          timeout_minutes: 2
+          max_attempts: 12
+          retry_wait_seconds: 5
+          command: "./tests/kind/test-metrics.sh 0"

      - name: Create reboot sentinel files
        run: |
          ./tests/kind/create-reboot-sentinels.sh

-      - name: Follow reboot until success
-        env:
-          DEBUG: true
-        run: |
-          ./tests/kind/follow-coordinated-reboot.sh
-
-  # This workflow is useful when introducing new versions, to ensure our manifests
-  # still work (even if there might be no manifest 'code' change).
-  # The version used here is what hasn't been tested with the helm chart
-  # This should therefore be "mid version" and (optionally) next new version
-  deploy-manifests:
-    name: Deploy kured with current manifests
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        kubernetes:
-          - "1.19"
-    steps:
-      - uses: actions/checkout@v2
-      - name: Find go version
-        run: |
-          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-          echo "::set-output name=version::${GO_VERSION}"
-        id: awk_gomod
-      - name: Ensure go version
-        uses: actions/setup-go@v2
-        with:
-          go-version: "${{ steps.awk_gomod.outputs.version }}"
-      - name: Build artifacts
-        run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" manifest
-      - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions
-        run: |
-          sudo bash << EOF
-              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
-              echo '{}' > /etc/docker/daemon.json
-              systemctl restart docker || journalctl --no-pager -n 500
-              systemctl status docker
-          EOF
-      # Default name for helm/kind-action kind clusters is "chart-testing"
-      - name: Create kind cluster
-        uses: helm/kind-action@master
-        with:
-          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
-      - name: Preload previously built images onto kind cluster
-        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
-      - name: Install kured with kubectl
-        run: |
-          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml
-      - name: Ensure kured is ready
+      - name: Get metrics (need reboot)
        uses: nick-invision/retry@v2.4.0
        with:
-          timeout_minutes: 10
+          timeout_minutes: 15
          max_attempts: 10
          retry_wait_seconds: 60
-          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
-          command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
+          command: "./tests/kind/test-metrics.sh 1"
+
+
+  # TEMPLATE Scenario testing.
+  # Note: keep in mind that the helm chart's appVersion is overriden to test your HEAD of the branch,
+  # if you `make helm-chart`.
+  # This will allow you to test properly your scenario and not use an existing image which will not
+  # contain your feature.
+
+  # scenario-<REPLACETHIS>-helm:
+  #   #example: Testing <REPLACETHIS> with helm chart and code from HEAD"
+  #   name: "<REPLACETHIS>"
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     fail-fast: false
+  #     # You can define your own kubernetes versions. For example if your helm chart change should behave differently with different kubernetes versions.
+  #     matrix:
+  #       kubernetes:
+  #         - "1.20"
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Find go version
+  #       run: |
+  #         GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
+  #         echo "::set-output name=version::${GO_VERSION}"
+  #       id: awk_gomod
+  #     - name: Ensure go version
+  #       uses: actions/setup-go@v2
+  #       with:
+  #         go-version: "${{ steps.awk_gomod.outputs.version }}"
+  #     - name: Build artifacts
+  #       run: |
+  #         make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
+  #         make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart
+  #
+  #     - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions"
+  #       run: |
+  #         sudo bash << EOF
+  #             cp /etc/docker/daemon.json /etc/docker/daemon.json.old
+  #             echo '{}' > /etc/docker/daemon.json
+  #             systemctl restart docker || journalctl --no-pager -n 500
+  #             systemctl status docker
+  #         EOF
+  #
+  #     # Default name for helm/kind-action kind clusters is "chart-testing"
+  #     - name: Create 5 node kind cluster
+  #       uses: helm/kind-action@master
+  #       with:
+  #         config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
+  #
+  #     - name: Preload previously built images onto kind cluster
+  #       run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
+  #
+  #     - name: Deploy kured on default namespace with its helm chart
+  #       run: |
+  #         # Documented in official helm doc to live on the edge
+  #         curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+  #         # Refresh bins
+  #         hash -r
+  #         helm install kured ./charts/kured/ --wait --values ./charts/kured/ci/<REPLACETHIS>-values.yaml
+  #         kubectl config set-context kind-chart-testing
+  #         kubectl get ds --all-namespaces
+  #         kubectl describe ds kured
+  #
+  #     - name: Ensure kured is ready
+  #       uses: nick-invision/retry@v2.4.0
+  #       with:
+  #         timeout_minutes: 10
+  #         max_attempts: 10
+  #         retry_wait_seconds: 60
+  #         # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = 5
+  #         command: "kubectl get ds kured | grep -E 'kured.*5.*5.*5.*5.*5' "
+  #
+  #     - name: Create reboot sentinel files
+  #       run: |
+  #         ./tests/kind/create-reboot-sentinels.sh
+  #
+  #     - name: Test <REPLACETHIS>
+  #       env:
+  #         DEBUG: true
+  #       run: |
+  #         <TODO>
@@ -25,7 +25,7 @@ jobs:
    steps:
    # Stale by default waits for 60 days before marking PR/issues as stale, and closes them after 21 days.
    # Do not expire the first issues that would allow the community to grow.
-    - uses: actions/stale@v3.0.18
+    - uses: actions/stale@v3.0.19
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        stale-issue-message: 'This issue was automatically considered stale due to lack of activity. Please update it and/or join our slack channels to promote it, before it automatically closes (in 7 days).'
@@ -68,15 +68,15 @@ jobs:
          image-name: docker.io/${{ github.repository_owner }}/kured:${{ github.sha }}

  deploy-helm:
-    name: Ensure a kubernetes change didn't break our code
+    name: Ensure our currently released helm chart works on all kubernetes versions
    runs-on: ubuntu-latest
    # only build with oldest and newest supported, it should be good enough.
    strategy:
      matrix:
        kubernetes:
-          - "1.18"
          - "1.19"
          - "1.20"
+          - "1.21"
    steps:
      - uses: actions/checkout@v2
      - name: Find go version
@@ -88,10 +88,6 @@ jobs:
        uses: actions/setup-go@v2
        with:
          go-version: "${{ steps.awk_gomod.outputs.version }}"
-      - name: Build artifacts
-        run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="main" image
-          make DH_ORG="${{ github.repository_owner }}" VERSION="main" helm-chart

      - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions"
        run: |
@@ -104,12 +100,10 @@ jobs:

      # Default name for helm/kind-action kind clusters is "chart-testing"
      - name: Create 5 node kind cluster
-        uses: helm/kind-action@master
+        uses: helm/kind-action@v1.1.0
        with:
          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
-
-      - name: Preload previously built images onto kind cluster
-        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:main --name chart-testing
+          version: v0.11.0

      - name: Deploy kured on default namespace with its helm chart
        run: |
@@ -13,7 +13,9 @@ you are planning to contribute code.
 [issues]: https://github.com/weaveworks/kured/issues
 [readme]: README.md

-## Updating k8s support
+## Regular development activities
+
+### Updating k8s support

 Whenever we want to update e.g. the `kubectl` or `client-go` dependencies,
 some RBAC changes might be necessary too.
@@ -24,15 +26,90 @@ This is what it took to support Kubernetes 1.14:
 That the process can be more involved that that can be seen in
 <https://github.com/weaveworks/kured/commits/support-k8s-1.10>

+Please update our .github/workflows with the new k8s images, starting by
+the creation of a .github/kind-cluster-<version>.yaml, then updating
+our workflows with the new versions.
+
 Once you updated everything, make sure you update the support matrix on
 the main [README][readme] as well.

-## Release testing
+### Updating other dependencies
+
+Dependabot proposes changes in our go.mod/go.sum.
+Some of those changes are covered by CI testing, some are not.
+
+Please make sure to test those not covered by CI (mostly the integration
+with other tools) manually before merging.
+
+### Review periodic jobs
+
+We run periodic jobs (see also Automated testing section of this documentation).
+Those should be monitored for failures.
+
+If a failure happen in periodics, something terribly wrong must have happened
+(or github is failing at the creation of a kind cluster). Please monitor those
+failures carefully.
+
+### Introducing new features
+
+When you introduce a new feature, the kured team expects you to have tested
+your change thoroughly. If possible, include all the necessary testing in your change.
+
+If your change involves a user facing change (change in flags of kured for example),
+please include expose your new feature in our default manifest (`kured-ds.yaml`),
+as a comment.
+
+Do not update the helm chart directly.
+Helm charts and our release manifests (see below) are our stable interfaces.
+Any user facing changes will therefore have to wait for a while before being
+exposed to our users.
+
+This also means that when you expose a new feature, you should create another PR
+for your changes in `charts/` to make your feature available for our next kured version.
+In this change, you can directly bump the appVersion to the next minor version.
+(for example, if current appVersion is 1.6.x, make sure you update your appVersion
+to 1.7.0). It allows us to have an easy view of what we land each release.
+
+Do not hesitate to increase the test coverage for your feature, whether it's unit
+testing to full functional testing (even using helm charts)
+
+### Increasing test coverage
+
+We are welcoming any change to increase our test coverage.
+See also our github issues for the label `testing`.
+
+### Updating helm charts
+
+Helm charts are continuously published. Any change in `charts/` will be immediately
+pushed in production.
+
+## Automated testing
+
+Our CI is covered by github actions.
+You can see their contents in .github/workflows.
+
+We currently run:
+- go tests and lint
+- shellcheck
+- a check for dead links in our docs
+- a security check against our base image (alpine)
+- a deep functional test using our manifests on all supported k8s versions
+- basic deployment using our helm chart on any chart change
+
+Changes in helm charts are not functionally tested on PRs. We assume that
+the PRs to implement the feature are properly tested by our users and
+contributors before merge.
+
+To test your code manually, follow the section Manual testing.
+
+## Manual (release) testing

 Before `kured` is released, we want to make sure it still works fine on the
 previous, current and next minor version of Kubernetes (with respect to the
 `client-go` & `kubectl` dependencies in use). For local testing e.g.
-`minikube` or `kind` can be sufficient.
+`minikube` or `kind` can be sufficient. This will allow you to catch issues
+that might not have been tested in our CI, like integration with other tools,
+or your specific use case.

 Deploy kured in your test scenario, make sure you pass the right `image`,
 update the e.g. `period` and `reboot-days` options, so you get immediate
@@ -42,7 +119,11 @@ results, if you login to a node and run:
 sudo touch /var/run/reboot-required
 ```

-### Testing with `minikube`
+### Example of golang testing
+
+Please run `make test`. You should have golint installed.
+
+### Example of testing with `minikube`

 A test-run with `minikube` could look like this:

@@ -82,7 +163,7 @@ If all the tests ran well, kured maintainers can reach out to the Weaveworks
 team to get an upcoming `kured` release tested in the Dev environment for
 real life testing.

-### Testing with `kind`
+### Example of testing with `kind`

 A test-run with `kind` could look like this:

@@ -1,5 +1,5 @@
 .DEFAULT: all
-.PHONY: all clean image publish-image minikube-publish manifest helm-chart
+.PHONY: all clean image publish-image minikube-publish manifest helm-chart test tests

 DH_ORG=weaveworks
 VERSION=$(shell git symbolic-ref --short HEAD)-$(shell git rev-parse --short HEAD)
@@ -43,3 +43,11 @@ helm-chart:
 	sed -i "s#appVersion:.*#appVersion: \"$(VERSION)\"#g" charts/kured/Chart.yaml
 	sed -i "s#\`[0-9]*\.[0-9]*\.[0-9]*\`#\`$(VERSION)\`#g" charts/kured/README.md
 	echo "Please bump version in charts/kured/Chart.yaml"
+
+test: tests
+	echo "Running go tests"
+	go test ./...
+	echo "Running golint on pkg"
+	golint ./pkg/...
+	echo "Running golint on cmd"
+	golint ./cmd/...
@@ -3,25 +3,27 @@

 <img src="https://github.com/weaveworks/kured/raw/main/img/logo.png" align="right"/>

-* [Introduction](#introduction)
-* [Kubernetes & OS Compatibility](#kubernetes-&-os-compatibility)
-* [Installation](#installation)
-* [Configuration](#configuration)
-  * [Reboot Sentinel File & Period](#reboot-sentinel-file-&-period)
-  * [Setting a schedule](#setting-a-schedule)
-  * [Blocking Reboots via Alerts](#blocking-reboots-via-alerts)
-  * [Blocking Reboots via Pods](#blocking-reboots-via-pods)
-  * [Prometheus Metrics](#prometheus-metrics)
-  * [Slack Notifications](#slack-notifications)
-  * [Overriding Lock Configuration](#overriding-lock-configuration)
-* [Operation](#operation)
-  * [Testing](#testing)
-  * [Disabling Reboots](#disabling-reboots)
-  * [Manual Unlock](#manual-unlock)
-  * [Automatic Unlock](#automatic-unlock)
-* [Building](#building)
-* [Frequently Asked/Anticipated Questions](#frequently-askedanticipated-questions)
-* [Getting Help](#getting-help)
+- [Introduction](#introduction)
+- [Kubernetes & OS Compatibility](#kubernetes--os-compatibility)
+- [Installation](#installation)
+- [Configuration](#configuration)
+  - [Reboot Sentinel File & Period](#reboot-sentinel-file--period)
+  - [Setting a schedule](#setting-a-schedule)
+  - [Blocking Reboots via Alerts](#blocking-reboots-via-alerts)
+  - [Blocking Reboots via Pods](#blocking-reboots-via-pods)
+  - [Prometheus Metrics](#prometheus-metrics)
+  - [Notifications](#notifications)
+  - [Overriding Lock Configuration](#overriding-lock-configuration)
+- [Operation](#operation)
+  - [Testing](#testing)
+  - [Disabling Reboots](#disabling-reboots)
+  - [Manual Unlock](#manual-unlock)
+  - [Automatic Unlock](#automatic-unlock)
+  - [Delaying Lock Release](#delaying-lock-release)
+- [Building](#building)
+- [Frequently Asked/Anticipated Questions](#frequently-askedanticipated-questions)
+  - [Why is there no `latest` tag on Docker Hub?](#why-is-there-no-latest-tag-on-docker-hub)
+- [Getting Help](#getting-help)

 ## Introduction

@@ -46,7 +48,8 @@ server:

 | kured | kubectl | k8s.io/client-go | k8s.io/apimachinery | expected kubernetes compatibility |
 |-------|---------|------------------|---------------------|-----------------------------------|
-| main  | 1.20.1  | v0.20.1          | v0.20.1             | 1.19.x, 1.20.x, 1.21.x            |
+| main  | 1.20.5  | v0.20.5          | v0.20.5             | 1.19.x, 1.20.x, 1.21.x            |
+| 1.7.0 | 1.20.5  | v0.20.5          | v0.20.5             | 1.19.x, 1.20.x, 1.21.x            |
 | 1.6.1 | 1.19.4  | v0.19.4          | v0.19.4             | 1.18.x, 1.19.x, 1.20.x            |
 | 1.5.1 | 1.18.8  | v0.18.8          | v0.18.8             | 1.17.x, 1.18.x, 1.19.x            |
 | 1.4.4 | 1.17.7  | v0.17.0          | v0.17.0             | 1.16.x, 1.17.x, 1.18.x            |
@@ -83,14 +86,19 @@ The following arguments can be passed to kured via the daemonset pod template:
 Flags:
      --alert-filter-regexp regexp.Regexp   alert names to ignore when checking for active alerts
      --blocking-pod-selector stringArray   label selector identifying pods whose presence should prevent reboots
+      --drain-grace-period int              time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1)
+      --skip-wait-for-delete-timeout int    when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0)
      --ds-name string                      name of daemonset on which to place lock (default "kured")
      --ds-namespace string                 namespace containing daemonset on which to place lock (default "kube-system")
      --end-time string                     schedule reboot only before this time of day (default "23:59:59")
+      --force-reboot bool                   force a reboot even if the drain is still running (default: false)
+      --drain-timeout duration              timeout after which the drain is aborted (default: 0, infinite time)
  -h, --help                                help for kured
      --lock-annotation string              annotation in which to record locking node (default "weave.works/kured-node-lock")
      --lock-ttl duration                   expire lock annotation after this duration (default: 0, disabled)
      --message-template-drain string       message template used to notify about a node being drained (default "Draining node %s")
      --message-template-reboot string      message template used to notify about a node being rebooted (default "Rebooting node %s")
+      --notify-url                          url for reboot notifications (cannot use with --slack-hook-url flags)
      --period duration                     reboot check period (default 1h0m0s)
      --prefer-no-schedule-taint string     Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Disabled by default. Set e.g. to "weave.works/kured-node-reboot" to enable tainting.
      --prometheus-url string               Prometheus instance to probe for active alerts
@@ -99,7 +107,7 @@ Flags:
      --reboot-sentinel string              path to file whose existence signals need to reboot (default "/var/run/reboot-required")
      --reboot-sentinel-command string      command for which a successful run signals need to reboot (default ""). If non-empty, sentinel file will be ignored.
      --slack-channel string                slack channel for reboot notfications
-      --slack-hook-url string               slack hook URL for reboot notfications
+      --slack-hook-url string               slack hook URL for reboot notfications [deprecated in favor of --notify-url]
      --slack-username string               slack username for reboot notfications (default "kured")
      --start-time string                   schedule reboot only after this time of day (default "0:00")
      --time-zone string                    use this timezone for schedule inputs (default "UTC")
@@ -234,14 +242,16 @@ Alternatively you can use the `--message-template-drain` and `--message-template

 Here is the syntax:

-slack:           `slack://tokenA/tokenB/tokenC`
+- slack:           `slack://tokenA/tokenB/tokenC`
 (`--slack-hook-url` is deprecated but possible to use)

-rocketchat:      `rocketchat://[username@]rocketchat-host/token[/channel|@recipient]`
+- rocketchat:      `rocketchat://[username@]rocketchat-host/token[/channel|@recipient]`

-teams:           `teams://token-a/token-b/token-c`
+- teams:           `teams://tName/token-a/token-b/token-c`

-Email:           `smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]`
+   > **Attention** as the [format of the url has changed](https://github.com/containrrr/shoutrrr/issues/138) you also have to specify a `tName`
+
+- Email:           `smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]`

 More details here: <https://github.com/containrrr/shoutrrr/blob/main/docs/services/overview.md>

@@ -301,6 +311,11 @@ which holds lock might be killed thus annotation will stay there for ever.

 Using `--lock-ttl=30m` will allow other nodes to take over if TTL has expired (in this case 30min) and continue reboot process.

+### Delaying Lock Release
+
+
+Using `--lock-release-delay=30m` will cause nodes to hold the lock for the specified time frame (in this case 30min) before it is released and the reboot process continues. This can be used to throttle reboots across the cluster.
+
 ## Building

 Kured now uses [Go
@@ -0,0 +1,13 @@
+# This is tested twice:
+# Basic install test with chart-testing (on charts PRs)
+# Functional testing in PRs (other PRs)
+
+service:
+  create: true
+  name: kured-prometheus-endpoint
+  port: 8080
+  type: NodePort
+  nodePort: 30000
+
+# Do not override the configuration: period in this, so that
+# We can test prometheus exposed metrics without rebooting.
@@ -16,11 +16,14 @@ metadata:
  {{- end }}
  {{- end }}
 spec:
-  type: ClusterIP
+  type: {{ .Values.service.type }}
  ports:
    - name: metrics
      port: {{ .Values.service.port }}
      targetPort: 8080
+      {{- if eq .Values.service.type "NodePort" }}
+      nodePort: {{ .Values.service.nodePort }}
+      {{- end }}
  selector:
    {{- include "kured.matchLabels" . | nindent 4 }}
 {{- end }}
@@ -65,6 +65,7 @@ service:
  port: 8080
  annotations: {}
  name: ""
+  type: ClusterIP

 podLabels: {}

@@ -38,24 +38,29 @@ var (
 	version = "unreleased"

 	// Command line flags
-	period                    time.Duration
-	dsNamespace               string
-	dsName                    string
-	lockAnnotation            string
-	lockTTL                   time.Duration
-	prometheusURL             string
-	preferNoScheduleTaintName string
-	alertFilter               *regexp.Regexp
-	rebootSentinelFile        string
-	rebootSentinelCommand     string
-	notifyURL                 string
-	slackHookURL              string
-	slackUsername             string
-	slackChannel              string
-	messageTemplateDrain      string
-	messageTemplateReboot     string
-	podSelectors              []string
-	rebootCommand             string
+	forceReboot                     bool
+	drainTimeout                    time.Duration
+	period                          time.Duration
+	drainGracePeriod                int
+	skipWaitForDeleteTimeoutSeconds int
+	dsNamespace                     string
+	dsName                          string
+	lockAnnotation                  string
+	lockTTL                         time.Duration
+	lockReleaseDelay                time.Duration  
+	prometheusURL                   string
+	preferNoScheduleTaintName       string
+	alertFilter                     *regexp.Regexp
+	rebootSentinelFile              string
+	rebootSentinelCommand           string
+	notifyURL                       string
+	slackHookURL                    string
+	slackUsername                   string
+	slackChannel                    string
+	messageTemplateDrain            string
+	messageTemplateReboot           string
+	podSelectors                    []string
+	rebootCommand                   string

 	rebootDays    []string
 	rebootStart   string
@@ -91,6 +96,14 @@ func main() {
 		PreRun: flagCheck,
 		Run:    root}

+	rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false,
+		"force a reboot even if the drain is still running (default: false)")
+	rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1,
+		"time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1)")
+	rootCmd.PersistentFlags().IntVar(&skipWaitForDeleteTimeoutSeconds, "skip-wait-for-delete-timeout", 0,
+		"when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0)")
+	rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 0,
+		"timeout after which the drain is aborted (default: 0, infinite time)")
 	rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60,
 		"sentinel check period")
 	rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system",
@@ -101,6 +114,8 @@ func main() {
 		"annotation in which to record locking node")
 	rootCmd.PersistentFlags().DurationVar(&lockTTL, "lock-ttl", 0,
 		"expire lock annotation after this duration (default: 0, disabled)")
+	rootCmd.PersistentFlags().DurationVar(&lockReleaseDelay, "lock-release-delay", 0,
+		"delay lock release for this duration (default: 0, disabled)")
 	rootCmd.PersistentFlags().StringVar(&prometheusURL, "prometheus-url", "",
 		"Prometheus instance to probe for active alerts")
 	rootCmd.PersistentFlags().Var(&regexpValue{&alertFilter}, "alert-filter-regexp",
@@ -308,6 +323,13 @@ func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}, TTL time.D
 	}
 }

+func throttle(releaseDelay time.Duration) {
+	if releaseDelay > 0 {
+		log.Infof("Delaying lock release by %v", releaseDelay)
+		time.Sleep(releaseDelay)
+	}
+}
+
 func release(lock *daemonsetlock.DaemonSetLock) {
 	log.Infof("Releasing lock")
 	if err := lock.Release(); err != nil {
@@ -332,20 +354,32 @@ func drain(client *kubernetes.Clientset, node *v1.Node) {
 	}

 	drainer := &kubectldrain.Helper{
-		Client:              client,
-		GracePeriodSeconds:  -1,
-		Force:               true,
-		DeleteEmptyDirData:  true,
-		IgnoreAllDaemonSets: true,
-		ErrOut:              os.Stderr,
-		Out:                 os.Stdout,
+		Client:                          client,
+		Ctx:                             context.Background(),
+		GracePeriodSeconds:              drainGracePeriod,
+		SkipWaitForDeleteTimeoutSeconds: skipWaitForDeleteTimeoutSeconds,
+		Force:                           true,
+		DeleteEmptyDirData:              true,
+		IgnoreAllDaemonSets:             true,
+		ErrOut:                          os.Stderr,
+		Out:                             os.Stdout,
+		Timeout:                         drainTimeout,
 	}
+
 	if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil {
-		log.Fatalf("Error cordonning %s: %v", nodename, err)
+		if !forceReboot {
+			log.Fatalf("Error cordonning %s: %v", nodename, err)
+		}
+		log.Errorf("Error cordonning %s: %v, continuing with reboot anyway", nodename, err)
+		return
 	}

 	if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil {
-		log.Fatalf("Error draining %s: %v", nodename, err)
+		if !forceReboot {
+			log.Fatalf("Error draining %s: %v", nodename, err)
+		}
+		log.Errorf("Error draining %s: %v, continuing with reboot anyway", nodename, err)
+		return
 	}
 }

@@ -436,7 +470,7 @@ func deleteNodeAnnotation(client *kubernetes.Clientset, nodeID, key string) {
 	}
 }

-func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []string, window *timewindow.TimeWindow, TTL time.Duration) {
+func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []string, window *timewindow.TimeWindow, TTL time.Duration, releaseDelay time.Duration) {
 	config, err := rest.InClusterConfig()
 	if err != nil {
 		log.Fatal(err)
@@ -468,6 +502,7 @@ func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []s
 				deleteNodeAnnotation(client, nodeID, KuredRebootInProgressAnnotation)
 			}
 		}
+		throttle(releaseDelay)
 		release(lock)
 	}

@@ -586,6 +621,11 @@ func root(cmd *cobra.Command, args []string) {
 	} else {
 		log.Info("Lock TTL not set, lock will remain until being released")
 	}
+	if lockReleaseDelay > 0 {
+		log.Infof("Lock release delay set, lock release will be delayed by: %v", lockReleaseDelay)
+	} else {
+		log.Info("Lock release delay not set, lock will be released immediately after rebooting")
+	}
 	log.Infof("PreferNoSchedule taint: %s", preferNoScheduleTaintName)
 	log.Infof("Blocking Pod Selectors: %v", podSelectors)
 	log.Infof("Reboot schedule: %v", window)
@@ -601,7 +641,7 @@ func root(cmd *cobra.Command, args []string) {
 	hostSentinelCommand := buildHostCommand(1, sentinelCommand)
 	hostRestartCommand := buildHostCommand(1, restartCommand)

-	go rebootAsRequired(nodeID, hostRestartCommand, hostSentinelCommand, window, lockTTL)
+	go rebootAsRequired(nodeID, hostRestartCommand, hostSentinelCommand, window, lockTTL, lockReleaseDelay)
 	go maintainRebootRequiredMetric(nodeID, hostSentinelCommand)

 	http.Handle("/metrics", promhttp.Handler())
@@ -3,15 +3,16 @@ module github.com/weaveworks/kured
 go 1.15

 require (
-	github.com/containrrr/shoutrrr v0.4.2
+	github.com/containrrr/shoutrrr v0.4.4
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
-	github.com/prometheus/client_golang v1.8.0
-	github.com/prometheus/common v0.15.0
+	github.com/prometheus/client_golang v1.10.0
+	github.com/prometheus/common v0.25.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.1.3
+	golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect
 	gotest.tools/v3 v3.0.3
-	k8s.io/api v0.20.1
-	k8s.io/apimachinery v0.20.1
-	k8s.io/client-go v0.20.1
-	k8s.io/kubectl v0.20.1
+	k8s.io/api v0.20.5
+	k8s.io/apimachinery v0.20.5
+	k8s.io/client-go v0.20.5
+	k8s.io/kubectl v0.20.5
 )
@@ -93,8 +93,8 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/containrrr/shoutrrr v0.4.2 h1:fAoWgO2DLbHeE3aMpeLJm/eZqZ/7hZ5aGoJAJZqGeOA=
-github.com/containrrr/shoutrrr v0.4.2/go.mod h1:zqL2BvfC1W4FujrT4b3/ZCLxvD+uoeEpBL7rg9Dqpbg=
+github.com/containrrr/shoutrrr v0.4.4 h1:vHZ4E/76pKVY+Jyn/qhBz3X540Bn8NI5ppPHK4PyILY=
+github.com/containrrr/shoutrrr v0.4.4/go.mod h1:zqL2BvfC1W4FujrT4b3/ZCLxvD+uoeEpBL7rg9Dqpbg=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -238,8 +238,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -313,6 +314,7 @@ github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZn
 github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -395,6 +397,7 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
@@ -459,8 +462,8 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.8.0 h1:zvJNkoCFAnYFNC24FV8nW4JdRJ3GIFcLbg65lL/JDcw=
-github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM=
+github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg=
+github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -474,17 +477,18 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
-github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM=
-github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
+github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
+github.com/prometheus/common v0.25.0 h1:IjJYZJCI8HZYtqA3xYwGyDzSCy1r4CA2GRh+4vdOmtE=
+github.com/prometheus/common v0.25.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -579,8 +583,9 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o=
+golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -642,8 +647,9 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -656,6 +662,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -702,10 +709,14 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -873,18 +884,18 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.20.1 h1:ud1c3W3YNzGd6ABJlbFfKXBKXO+1KdGfcgGGNgFR03E=
-k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
-k8s.io/apimachinery v0.20.1 h1:LAhz8pKbgR8tUwn7boK+b2HZdt7MiTu2mkYtFMUjTRQ=
-k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/cli-runtime v0.20.1 h1:fJhRQ9EfTpJpCqSFOAqnYLuu5aAM7yyORWZ26qW1jJc=
-k8s.io/cli-runtime v0.20.1/go.mod h1:6wkMM16ZXTi7Ow3JLYPe10bS+XBnIkL6V9dmEz0mbuY=
-k8s.io/client-go v0.20.1 h1:Qquik0xNFbK9aUG92pxHYsyfea5/RPO9o9bSywNor+M=
-k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
-k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg=
-k8s.io/component-base v0.20.1 h1:6OQaHr205NSl24t5wOF2IhdrlxZTWEZwuGlLvBgaeIg=
-k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
-k8s.io/component-helpers v0.20.1/go.mod h1:Q8trCj1zyLNdeur6pD2QvsF8d/nWVfK71YjN5+qVXy4=
+k8s.io/api v0.20.5 h1:zsMTffV0Le2EiI0aKvlTHEnXGxk1HiqGRhJcCPiI7JI=
+k8s.io/api v0.20.5/go.mod h1:FQjAceXnVaWDeov2YUWhOb6Yt+5UjErkp6UO3nczO1Y=
+k8s.io/apimachinery v0.20.5 h1:wO/FxMVRn223rAKxnBbwCyuN96bS9MFTIvP0e/V7cps=
+k8s.io/apimachinery v0.20.5/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
+k8s.io/cli-runtime v0.20.5 h1:VIT1Y6ty6TsZaKXiyqH94U+7vR0pUpBQ1UCrTNoS+UU=
+k8s.io/cli-runtime v0.20.5/go.mod h1:ihjPeQWDk7NGVIkNEvpwxA3gJvqtU+LtkDj11TvyXn4=
+k8s.io/client-go v0.20.5 h1:dJGtYUvFrFGjQ+GjXEIby0gZWdlAOc0xJBJqY3VyDxA=
+k8s.io/client-go v0.20.5/go.mod h1:Ee5OOMMYvlH8FCZhDsacjMlCBwetbGZETwo1OA+e6Zw=
+k8s.io/code-generator v0.20.5/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg=
+k8s.io/component-base v0.20.5 h1:8BZQKLJGhWrxtB7kIOEejKDtAKr1HOYvB0PZNeTyLS0=
+k8s.io/component-base v0.20.5/go.mod h1:l0isoBLGyQKwRoTWbPHR6jNDd3/VqQD43cNlsjddGng=
+k8s.io/component-helpers v0.20.5/go.mod h1:AzTdoPj6YAN2SUfhBX/FUUU3ntfFuse03q/VMLovEsE=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
@@ -893,9 +904,9 @@ k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
-k8s.io/kubectl v0.20.1 h1:7h1vSrL/B3hLrhlCJhbTADElPKDbx+oVUt3+QDSXxBo=
-k8s.io/kubectl v0.20.1/go.mod h1:2bE0JLYTRDVKDiTREFsjLAx4R2GvUtL/mGYFXfFFMzY=
-k8s.io/metrics v0.20.1/go.mod h1:JhpBE/fad3yRGsgEpiZz5FQQM5wJ18OTLkD7Tv40c0s=
+k8s.io/kubectl v0.20.5 h1:/wndy8hw5TsL8G8KWPDJrtPKS8D34uSdWS0BMRmtzWs=
+k8s.io/kubectl v0.20.5/go.mod h1:mlNQgyV18D4XFt5BmfSkrxQNS+arT2pXDQxxnH5lMiw=
+k8s.io/metrics v0.20.5/go.mod h1:vsptOayjKWKWHvWR1vFQY++vxydzaEo/2+JC7kSDKPU=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 mvdan.cc/sh v2.6.4+incompatible/go.mod h1:IeeQbZq+x2SUGBensq/jge5lLQbS3XT2ktyp3wrt4x8=
@@ -44,23 +44,32 @@ spec:
                  fieldPath: spec.nodeName
          command:
            - /usr/bin/kured
-#            - --alert-filter-regexp=^RebootRequired$
-#            - --blocking-pod-selector=runtime=long,cost=expensive
-#            - --blocking-pod-selector=name=temperamental
-#            - --blocking-pod-selector=...
-#            - --ds-name=kured
-#            - --ds-namespace=kube-system
-#            - --end-time=23:59:59
-#            - --lock-annotation=weave.works/kured-node-lock
+#            - --force-reboot=false
+#            - --drain-grace-period=-1
+#            - --skip-wait-for-delete-timeout=0
+#            - --drain-timeout=0
 #            - --period=1h
+#            - --ds-namespace=kube-system
+#            - --ds-name=kured
+#            - --lock-annotation=weave.works/kured-node-lock
+#            - --lock-ttl=0
 #            - --prometheus-url=http://prometheus.monitoring.svc.cluster.local
-#            - --reboot-days=sun,mon,tue,wed,thu,fri,sat
+#            - --alert-filter-regexp=^RebootRequired$
 #            - --reboot-sentinel=/var/run/reboot-required
+#            - --prefer-no-schedule-taint=""
+#            - --reboot-sentinel-command=""
 #            - --slack-hook-url=https://hooks.slack.com/...
 #            - --slack-username=prod
 #            - --slack-channel=alerting
+#            - --notify-url="" # See also shoutrrr url format
 #            - --message-template-drain=Draining node %s
 #            - --message-template-drain=Rebooting node %s
+#            - --blocking-pod-selector=runtime=long,cost=expensive
+#            - --blocking-pod-selector=name=temperamental
+#            - --blocking-pod-selector=...
+#            - --reboot-days=sun,mon,tue,wed,thu,fri,sat
 #            - --start-time=0:00
+#            - --end-time=23:59:59
 #            - --time-zone=UTC
 #            - --annotate-nodes=false
+#            - --lock-release-delay=30m
@@ -6,11 +6,18 @@ import (
 	"fmt"
 	"time"

+	v1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 )

+const (
+	k8sAPICallRetrySleep   = 5 * time.Second // How much time to wait in between retrying a k8s API call
+	k8sAPICallRetryTimeout = 5 * time.Minute // How long to wait until we determine that the k8s API is definitively unavailable
+)
+
 // DaemonSetLock holds all necessary information to do actions
 // on the kured ds which holds lock info through annotations.
 type DaemonSetLock struct {
@@ -34,11 +41,11 @@ func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation strin
 }

 // Acquire attempts to annotate the kured daemonset with lock info from instantiated DaemonSetLock using client-go
-func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acquired bool, owner string, err error) {
+func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (bool, string, error) {
 	for {
-		ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{})
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
-			return false, "", err
+			return false, "", fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 		}

 		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
@@ -78,10 +85,10 @@ func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acqu
 }

 // Test attempts to check the kured daemonset lock status (existence, expiry) from instantiated DaemonSetLock using client-go
-func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) {
-	ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{})
+func (dsl *DaemonSetLock) Test(metadata interface{}) (bool, error) {
+	ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 	if err != nil {
-		return false, err
+		return false, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 	}

 	valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
@@ -102,9 +109,9 @@ func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) {
 // Release attempts to remove the lock data from the kured ds annotations using client-go
 func (dsl *DaemonSetLock) Release() error {
 	for {
-		ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{})
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
-			return err
+			return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 		}

 		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
@@ -137,6 +144,24 @@ func (dsl *DaemonSetLock) Release() error {
 	}
 }

+// GetDaemonSet returns the named DaemonSet resource from the DaemonSetLock's configured client
+func (dsl *DaemonSetLock) GetDaemonSet(sleep, timeout time.Duration) (*v1.DaemonSet, error) {
+	var ds *v1.DaemonSet
+	var lastError error
+	err := wait.PollImmediate(sleep, timeout, func() (bool, error) {
+		ctx, cancel := context.WithTimeout(context.Background(), timeout)
+		defer cancel()
+		if ds, lastError = dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(ctx, dsl.name, metav1.GetOptions{}); lastError != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Timed out trying to get daemonset %s in namespace %s: %v", dsl.name, dsl.namespace, lastError)
+	}
+	return ds, nil
+}
+
 func ttlExpired(created time.Time, ttl time.Duration) bool {
 	if ttl > 0 && time.Since(created) >= ttl {
 		return true
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+expected="$1"
+if [[ "$expected" != "0" && "$expected" != "1" ]]; then
+    echo "You should give an argument to this script, the gauge value (0 or 1)"
+    exit 1
+fi
+
+HOST="${HOST:-localhost}"
+PORT="${PORT:-30000}"
+NODENAME="${NODENAME-chart-testing-control-plane}"
+
+reboot_required=$(docker exec "$NODENAME" curl "http://$HOST:$PORT/metrics" | awk '/^kured_reboot_required/{print $2}')
+if [[ "$reboot_required" == "$expected" ]]; then
+    echo "Test success"
+else
+    echo "Test failed"
+    exit 1
+fi