Requested changes to multi-arch build

build(deps): bump helm/chart-testing-action from 2.1.0 to 2.2.0

.github/ct.yaml

@@ -1,5 +1,6 @@
 # See https://github.com/helm/chart-testing#configuration
 remote: origin
+target-branch: main
 chart-dirs:
   - charts
 chart-repos: []

.github/dependabot.yml

@@ -15,3 +15,7 @@ updates:
       - dependency-name: "k8s.io/apimachinery"
       - dependency-name: "k8s.io/client-go"
       - dependency-name: "k8s.io/kubectl"
+  - package-ecosystem: "docker"
+    directory: "cmd/kured"
+    schedule:
+      interval: "daily"

.github/kind-cluster-1.20.yaml

@@ -1,13 +0,0 @@
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  image: "kindest/node:v1.20.0"
-- role: control-plane
-  image: "kindest/node:v1.20.0"
-- role: control-plane
-  image: "kindest/node:v1.20.0"
-- role: worker
-  image: "kindest/node:v1.20.0"
-- role: worker
-  image: "kindest/node:v1.20.0"

.github/kind-cluster-1.21.yaml

@@ -2,12 +2,12 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
-  image: kindest/node:v1.18.8
+  image: kindest/node:v1.21.2
 - role: control-plane
-  image: kindest/node:v1.18.8
+  image: kindest/node:v1.21.2
 - role: control-plane
-  image: kindest/node:v1.18.8
+  image: kindest/node:v1.21.2
 - role: worker
-  image: kindest/node:v1.18.8
+  image: kindest/node:v1.21.2
 - role: worker
-  image: kindest/node:v1.18.8
+  image: kindest/node:v1.21.2

.github/kind-cluster-1.22.yaml

@@ -2,12 +2,12 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
-  image: kindest/node:v1.19.4
+  image: kindest/node:v1.22.4
 - role: control-plane
-  image: kindest/node:v1.19.4
+  image: kindest/node:v1.22.4
 - role: control-plane
-  image: kindest/node:v1.19.4
+  image: kindest/node:v1.22.4
 - role: worker
-  image: kindest/node:v1.19.4
+  image: kindest/node:v1.22.4
 - role: worker
-  image: kindest/node:v1.19.4
+  image: kindest/node:v1.22.4

.github/kind-cluster-1.23.yaml

@@ -0,0 +1,13 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: "kindest/node:v1.23.0"
+- role: control-plane
+  image: "kindest/node:v1.23.0"
+- role: control-plane
+  image: "kindest/node:v1.23.0"
+- role: worker
+  image: "kindest/node:v1.23.0"
+- role: worker
+  image: "kindest/node:v1.23.0"

.github/workflows/on-main-push-charts.yaml

@@ -2,7 +2,7 @@ name: Publish helm chart
 on:
   push:
     branches:
-      - "master"
+      - "main"
     paths:
       - "charts/**"
 

.github/workflows/on-main-push.yaml

@@ -0,0 +1,59 @@
+# We publish every merged commit in the form of an image
+# named kured:<branch>-<short tag>
+name: Push image of latest main
+on:
+  push:
+    branches:
+      - main
+jobs:
+  tag-scan-and-push-final-image:
+    name: "Build, scan, and publish tagged image"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Find go version
+        run: |
+          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
+          echo "::set-output name=version::${GO_VERSION}"
+        id: awk_gomod
+
+      - name: Ensure go version
+        uses: actions/setup-go@v2
+        with:
+          go-version: "${{ steps.awk_gomod.outputs.version }}"
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_WEAVEWORKSKUREDCI }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_WEAVEWORKSKUREDCI }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: weave-ghcr-bot
+          password: ${{ secrets.KURED_WEAVE_GHCR_BOT_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Find current tag version
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+        id: tags
+
+      - name: Build image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: cmd/kured/Dockerfile.multi
+          platforms: linux/arm64, linux/amd64
+          push: true
+          tags: |
+            docker.io/${{ GITHUB.REPOSITORY }}:main-${{ steps.tags.outputs.sha_short }}
+            ghcr.io/${{ GITHUB.REPOSITORY }}:main-${{ steps.tags.outputs.sha_short }}

.github/workflows/on-master-push.yaml

@@ -1,38 +0,0 @@
-# We publish every merged commit in the form of an image
-# named kured:<branch>-<short tag>
-name: Push image of latest master
-on:
-  push:
-    branches:
-      - master
-jobs:
-  tag-scan-and-push-final-image:
-    name: "Build, scan, and publish tagged image"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-
-      - name: Find go version
-        run: |
-          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-          echo "::set-output name=version::${GO_VERSION}"
-        id: awk_gomod
-
-      - name: Ensure go version
-        uses: actions/setup-go@v2
-        with:
-          go-version: "${{ steps.awk_gomod.outputs.version }}"
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME_WEAVEWORKSKUREDCI }}
-          password: ${{ secrets.DOCKERHUB_TOKEN_WEAVEWORKSKUREDCI }}
-
-      - name: Build image
-        run: |
-          make DH_ORG="${{ github.repository_owner }}" image
-
-      - name: Publish image
-        run: |
-          make DH_ORG="${{ github.repository_owner }}" publish-image

.github/workflows/on-pr-charts.yaml

@@ -1,5 +1,6 @@
 #This is just extra testing, for lint check, and basic installation
-#If those fail, no need to test the rest of the PR (github will cancel the rest of the builds)
+#Those can fail earlier than functional tests (shorter tests)
+# and give developer feedback soon if they didn't test themselves
 name: PR - charts
 on:
   pull_request:
@@ -11,7 +12,7 @@ jobs:
   # tackling that for us.
   # Fail-fast ensures that if one of those matrix job fail, the other one gets cancelled.
   test-chart:
-    name: Test helm chart
+    name: Test helm chart changes
     runs-on: ubuntu-latest
     strategy:
       fail-fast: true
@@ -31,11 +32,47 @@ jobs:
 
       # Helm is already present in github actions, so do not re-install it
       - name: Setup chart testing
-        uses: helm/chart-testing-action@v2.0.1
+        uses: helm/chart-testing-action@v2.2.0
 
       - name: Create default kind cluster
-        uses: helm/kind-action@v1.1.0
+        uses: helm/kind-action@v1.2.0
+        with:
+          version: v0.11.0
         if: ${{ matrix.test-action == 'install' }}
 
-      - name: Run chart tests 
+      - name: Run chart tests
         run: ct ${{ matrix.test-action }} --config .github/ct.yaml
+
+  # This doesn't re-use the ct actions, due to many limitations (auto tear down, no real testing)
+  deploy-chart:
+    name: Functional test of helm chart in its current state (needs published image of the helm chart)
+    runs-on: ubuntu-latest
+    needs: test-chart
+    steps:
+      - uses: actions/checkout@v2
+      
+      # Default name for helm/kind-action kind clusters is "chart-testing"
+      - name: Create 1 node kind cluster
+        uses: helm/kind-action@v1.2.0
+        with:
+          version: v0.11.0
+
+      - name: Deploy kured on default namespace with its helm chart
+        run: |
+          # Documented in official helm doc to live on the edge
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          # Refresh bins
+          hash -r
+          helm install kured ./charts/kured/ --set configuration.period=1m --wait
+          kubectl config set-context kind-chart-testing
+          kubectl get ds --all-namespaces
+          kubectl describe ds kured
+
+      - name: Test if successful deploy
+        uses: nick-invision/retry@v2.6.0
+        with:
+          timeout_minutes: 10
+          max_attempts: 10
+          retry_wait_seconds: 10
+          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
+          command: "kubectl get ds kured | grep -E 'kured.*1.*1.*1.*1.*1'"

.github/workflows/on-pr.yaml

@@ -4,6 +4,29 @@ on:
   push:
 
 jobs:
+  pr-gotest:
+    name: Run go tests
+    runs-on: ubuntu-18.04
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Find go version
+        run: |
+          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
+          echo "::set-output name=version::${GO_VERSION}"
+        id: awk_gomod
+      - name: Ensure go version
+        uses: actions/setup-go@v2
+        with:
+          go-version: "${{ steps.awk_gomod.outputs.version }}"
+      - name: run tests
+        run: go test -json ./... > test.json
+      - name: Annotate tests
+        if: always()
+        uses: guyarb/golang-test-annoations@v0.5.0
+        with:
+          test-results: test.json
+
   pr-shellcheck:
     name: Lint bash code with shellcheck
     runs-on: ubuntu-latest
@@ -39,7 +62,7 @@ jobs:
     name: Check docs for incorrect links
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: Link Checker
       id: lc
       uses: peter-evans/link-checker@v1
@@ -70,16 +93,93 @@ jobs:
         with:
           image-name: docker.io/${{ github.repository_owner }}/kured:${{ github.sha }}
 
-  # If the PRs don't break the behaviour in the helm chart, we can simply publish the helm chart at the time of the release.
-  e2e-helm:
-    name: "Functional test of helm chart, e2e testing"
+  # This ensures the latest code works with the manifests built from tree.
+  # It is useful for two things:
+  # - Test manifests changes (obviously), ensuring they don't break existing clusters
+  # - Ensure manifests work with the latest versions even with no manifest change
+  #     (compared to helm charts, manifests cannot easily template changes based on versions)
+  # Helm charts are _trailing_ releases, while manifests are done during development.
+  e2e-manifests:
+    name: End-to-End test with kured with code and manifests from HEAD
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        kubernetes:
+          - "1.21"
+          - "1.22"
+          - "1.23"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Find go version
+        run: |
+          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
+          echo "::set-output name=version::${GO_VERSION}"
+        id: awk_gomod
+      - name: Ensure go version
+        uses: actions/setup-go@v2
+        with:
+          go-version: "${{ steps.awk_gomod.outputs.version }}"
+      - name: Build artifacts
+        run: |
+          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
+          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" manifest
+
+      - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions
+        run: |
+          sudo bash << EOF
+              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
+              echo '{}' > /etc/docker/daemon.json
+              systemctl restart docker || journalctl --no-pager -n 500
+              systemctl status docker
+          EOF
+
+      # Default name for helm/kind-action kind clusters is "chart-testing"
+      - name: Create kind cluster with 5 nodes
+        uses: helm/kind-action@v1.2.0
+        with:
+          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
+          version: v0.11.0
+
+      - name: Preload previously built images onto kind cluster
+        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
+
+      - name: Do not wait for an hour before detecting the rebootSentinel
+        run: |
+          sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds.yaml
+
+      - name: Install kured with kubectl
+        run: |
+          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml
+
+      - name: Ensure kured is ready
+        uses: nick-invision/retry@v2.6.0
+        with:
+          timeout_minutes: 10
+          max_attempts: 10
+          retry_wait_seconds: 60
+          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
+          command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
+
+      - name: Create reboot sentinel files
+        run: |
+          ./tests/kind/create-reboot-sentinels.sh
+
+      - name: Follow reboot until success
+        env:
+          DEBUG: true
+        run: |
+          ./tests/kind/follow-coordinated-reboot.sh
+
+  scenario-prom-helm:
+    name: Test prometheus with latest code from HEAD (=overrides image of the helm chart)
     runs-on: ubuntu-latest
     # only build with oldest and newest supported, it should be good enough.
     strategy:
+      fail-fast: false
       matrix:
         kubernetes:
-          - "1.18"
-          - "1.20"
+          - "1.21"
     steps:
       - uses: actions/checkout@v2
       - name: Find go version
@@ -96,7 +196,7 @@ jobs:
           make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
           make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart
 
-      - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions"
+      - name: Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions
         run: |
           sudo bash << EOF
               cp /etc/docker/daemon.json /etc/docker/daemon.json.old
@@ -106,10 +206,10 @@ jobs:
           EOF
 
       # Default name for helm/kind-action kind clusters is "chart-testing"
-      - name: Create 5 node kind cluster
-        uses: helm/kind-action@master
+      - name: Create 1 node kind cluster
+        uses: helm/kind-action@v1.2.0
         with:
-          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
+          version: v0.11.0
 
       - name: Preload previously built images onto kind cluster
         run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
@@ -117,81 +217,120 @@ jobs:
       - name: Deploy kured on default namespace with its helm chart
         run: |
           # Documented in official helm doc to live on the edge
-          curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
           # Refresh bins
           hash -r
-          helm install kured ./charts/kured/ --set configuration.period=1m
+          helm install kured ./charts/kured/ --wait --values ./charts/kured/ci/prometheus-values.yaml
           kubectl config set-context kind-chart-testing
           kubectl get ds --all-namespaces
           kubectl describe ds kured
 
       - name: Ensure kured is ready
-        uses: nick-invision/retry@v2.4.0
+        uses: nick-invision/retry@v2.6.0
         with:
           timeout_minutes: 10
           max_attempts: 10
           retry_wait_seconds: 60
-          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = 5
-          command: "kubectl get ds kured | grep -E 'kured.*5.*5.*5.*5.*5' "
+          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE
+          command: "kubectl get ds kured | grep -E 'kured.*1.*1.*1.*1.*1' "
+
+      - name: Get metrics (healthy)
+        uses: nick-invision/retry@v2.6.0
+        with:
+          timeout_minutes: 2
+          max_attempts: 12
+          retry_wait_seconds: 5
+          command: "./tests/kind/test-metrics.sh 0"
 
       - name: Create reboot sentinel files
         run: |
           ./tests/kind/create-reboot-sentinels.sh
 
-      - name: Follow reboot until success
-        env:
-          DEBUG: true
-        run: |
-          ./tests/kind/follow-coordinated-reboot.sh
-
-  # This workflow is useful when introducing new versions, to ensure our manifests
-  # still work (even if there might be no manifest 'code' change).
-  # The version used here is what hasn't been tested with the helm chart
-  deploy-manifests:
-    name: Deploy kured with current manifests
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        kubernetes:
-          - 1.18
-    steps:
-      - uses: actions/checkout@v2
-      - name: Find go version
-        run: |
-          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-          echo "::set-output name=version::${GO_VERSION}"
-        id: awk_gomod
-      - name: Ensure go version
-        uses: actions/setup-go@v2
+      - name: Get metrics (need reboot)
+        uses: nick-invision/retry@v2.6.0
         with:
-          go-version: "${{ steps.awk_gomod.outputs.version }}"
-      - name: Build artifacts
-        run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" manifest
-      - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions
-        run: |
-          sudo bash << EOF
-              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
-              echo '{}' > /etc/docker/daemon.json
-              systemctl restart docker || journalctl --no-pager -n 500
-              systemctl status docker
-          EOF
-      # Default name for helm/kind-action kind clusters is "chart-testing"
-      - name: Create kind cluster
-        uses: helm/kind-action@master
-        with:
-          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
-      - name: Preload previously built images onto kind cluster
-        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
-      - name: Install kured with kubectl
-        run: |
-          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml
-      - name: Ensure kured is ready
-        uses: nick-invision/retry@v2.4.0
-        with:
-          timeout_minutes: 10
+          timeout_minutes: 15
           max_attempts: 10
           retry_wait_seconds: 60
-          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
-          command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
+          command: "./tests/kind/test-metrics.sh 1"
+
+
+  # TEMPLATE Scenario testing.
+  # Note: keep in mind that the helm chart's appVersion is overriden to test your HEAD of the branch,
+  # if you `make helm-chart`.
+  # This will allow you to test properly your scenario and not use an existing image which will not
+  # contain your feature.
+
+  # scenario-<REPLACETHIS>-helm:
+  #   #example: Testing <REPLACETHIS> with helm chart and code from HEAD"
+  #   name: "<REPLACETHIS>"
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     fail-fast: false
+  #     # You can define your own kubernetes versions. For example if your helm chart change should behave differently with different kubernetes versions.
+  #     matrix:
+  #       kubernetes:
+  #         - "1.20"
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Find go version
+  #       run: |
+  #         GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
+  #         echo "::set-output name=version::${GO_VERSION}"
+  #       id: awk_gomod
+  #     - name: Ensure go version
+  #       uses: actions/setup-go@v2
+  #       with:
+  #         go-version: "${{ steps.awk_gomod.outputs.version }}"
+  #     - name: Build artifacts
+  #       run: |
+  #         make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
+  #         make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart
+  #
+  #     - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions"
+  #       run: |
+  #         sudo bash << EOF
+  #             cp /etc/docker/daemon.json /etc/docker/daemon.json.old
+  #             echo '{}' > /etc/docker/daemon.json
+  #             systemctl restart docker || journalctl --no-pager -n 500
+  #             systemctl status docker
+  #         EOF
+  #
+  #     # Default name for helm/kind-action kind clusters is "chart-testing"
+  #     - name: Create 5 node kind cluster
+  #       uses: helm/kind-action@master
+  #       with:
+  #         config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
+  #
+  #     - name: Preload previously built images onto kind cluster
+  #       run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
+  #
+  #     - name: Deploy kured on default namespace with its helm chart
+  #       run: |
+  #         # Documented in official helm doc to live on the edge
+  #         curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+  #         # Refresh bins
+  #         hash -r
+  #         helm install kured ./charts/kured/ --wait --values ./charts/kured/ci/<REPLACETHIS>-values.yaml
+  #         kubectl config set-context kind-chart-testing
+  #         kubectl get ds --all-namespaces
+  #         kubectl describe ds kured
+  #
+  #     - name: Ensure kured is ready
+  #       uses: nick-invision/retry@v2.6.0
+  #       with:
+  #         timeout_minutes: 10
+  #         max_attempts: 10
+  #         retry_wait_seconds: 60
+  #         # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = 5
+  #         command: "kubectl get ds kured | grep -E 'kured.*5.*5.*5.*5.*5' "
+  #
+  #     - name: Create reboot sentinel files
+  #       run: |
+  #         ./tests/kind/create-reboot-sentinels.sh
+  #
+  #     - name: Test <REPLACETHIS>
+  #       env:
+  #         DEBUG: true
+  #       run: |
+  #         <TODO>

.github/workflows/on-tag.yaml

@@ -12,7 +12,7 @@ jobs:
     name: "Build, scan, and publish tagged image"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
       - name: Find go version
         run: |
           GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
@@ -22,12 +22,14 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: "${{ steps.awk_gomod.outputs.version }}"
+      - name: Find current tag version
+        run: echo "::set-output name=version::${GITHUB_REF#refs/tags/}"
+        id: tags
       - run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${GITHUB_REF#refs/tags/}" image
-
+          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ steps.tags.outputs.version }}" image
       - uses: Azure/container-scan@v0
         with:
-          image-name: docker.io/${{ github.repository_owner }}/kured:${GITHUB_REF#refs/tags/}
+          image-name: docker.io/${{ github.repository_owner }}/kured:${{ steps.tags.outputs.version }}
 
       - name: Login to DockerHub
         uses: docker/login-action@v1
@@ -35,6 +37,27 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME_WEAVEWORKSKUREDCI }}
           password: ${{ secrets.DOCKERHUB_TOKEN_WEAVEWORKSKUREDCI }}
 
-      - name: Publish image
-        run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${GITHUB_REF#refs/tags/}" publish-image
+      - name: Login to ghcr.io
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: weave-ghcr-bot
+          password: ${{ secrets.KURED_WEAVE_GHCR_BOT_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: cmd/kured/Dockerfile.multi
+          platforms: linux/arm64, linux/amd64, linux/arm/v7, linux/arm/v6, linux/386
+          push: true
+          tags: |
+            docker.io/${{ GITHUB.REPOSITORY }}:${{ steps.tags.outputs.version }}
+            ghcr.io/${{ GITHUB.REPOSITORY }}:${{ steps.tags.outputs.version }}

.github/workflows/periodics-daily.yaml

@@ -5,26 +5,41 @@ on:
   - cron: "30 1 * * *"
 
 jobs:
+  periodics-gotest:
+    name: Run go tests
+    runs-on: ubuntu-18.04
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: run tests
+        run: go test -json ./... > test.json
+      - name: Annotate tests
+        if: always()
+        uses: guyarb/golang-test-annoations@v0.5.0
+        with:
+          test-results: test.json
+
   periodics-mark-stale:
     name: Mark stale issues and PRs
     runs-on: ubuntu-latest
     steps:
-    # Stale by default waits for 60 days before marking PR/issues as stale, and closes them after 7 days.
+    # Stale by default waits for 60 days before marking PR/issues as stale, and closes them after 21 days.
     # Do not expire the first issues that would allow the community to grow.
-    - uses: actions/stale@v3.0.14
+    - uses: actions/stale@v4
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue was automatically considered stale due to lack of activity. Please update it and/or join our slack channels to promote it, before it automatically closes (in 7 days).'
         stale-pr-message: 'This PR was automatically considered stale due to lack of activity. Please refresh it and/or join our slack channels to highlight it, before it automatically closes (in 7 days).'
         stale-issue-label: 'no-issue-activity'
         stale-pr-label: 'no-pr-activity'
-        exempt-issue-labels: 'good-first-issue'
+        exempt-issue-labels: 'good first issue,keep'
+        days-before-close: 21
 
   check-docs-links:
     name: Check docs for incorrect links
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: Link Checker
       id: lc
       uses: peter-evans/link-checker@v1
@@ -53,15 +68,15 @@ jobs:
           image-name: docker.io/${{ github.repository_owner }}/kured:${{ github.sha }}
 
   deploy-helm:
-    name: Ensure a kubernetes change didn't break our code
+    name: Ensure our currently released helm chart works on all kubernetes versions
     runs-on: ubuntu-latest
     # only build with oldest and newest supported, it should be good enough.
     strategy:
       matrix:
         kubernetes:
-          - 1.18
-          - 1.19
-          - 1.20
+          - "1.21"
+          - "1.22"
+          - "1.23"
     steps:
       - uses: actions/checkout@v2
       - name: Find go version
@@ -73,10 +88,6 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: "${{ steps.awk_gomod.outputs.version }}"
-      - name: Build artifacts
-        run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="master" image
-          make DH_ORG="${{ github.repository_owner }}" VERSION="master" helm-chart
 
       - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions"
         run: |
@@ -89,17 +100,15 @@ jobs:
 
       # Default name for helm/kind-action kind clusters is "chart-testing"
       - name: Create 5 node kind cluster
-        uses: helm/kind-action@master
+        uses: helm/kind-action@v1.2.0
         with:
           config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
-
-      - name: Preload previously built images onto kind cluster
-        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:master --name chart-testing
+          version: v0.11.0
 
       - name: Deploy kured on default namespace with its helm chart
         run: |
           # Documented in official helm doc to live on the edge
-          curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
           # Refresh bins
           hash -r
           helm install kured ./charts/kured/ --set configuration.period=1m
@@ -108,7 +117,7 @@ jobs:
           kubectl describe ds kured
 
       - name: Ensure kured is ready
-        uses: nick-invision/retry@v2.4.0
+        uses: nick-invision/retry@v2.6.0
         with:
           timeout_minutes: 10
           max_attempts: 10

DEVELOPMENT.md

@@ -13,7 +13,9 @@ you are planning to contribute code.
 [issues]: https://github.com/weaveworks/kured/issues
 [readme]: README.md
 
-## Updating k8s support
+## Regular development activities
+
+### Updating k8s support
 
 Whenever we want to update e.g. the `kubectl` or `client-go` dependencies,
 some RBAC changes might be necessary too.
@@ -24,15 +26,90 @@ This is what it took to support Kubernetes 1.14:
 That the process can be more involved that that can be seen in
 <https://github.com/weaveworks/kured/commits/support-k8s-1.10>
 
+Please update our .github/workflows with the new k8s images, starting by
+the creation of a .github/kind-cluster-<version>.yaml, then updating
+our workflows with the new versions.
+
 Once you updated everything, make sure you update the support matrix on
 the main [README][readme] as well.
 
-## Release testing
+### Updating other dependencies
+
+Dependabot proposes changes in our go.mod/go.sum.
+Some of those changes are covered by CI testing, some are not.
+
+Please make sure to test those not covered by CI (mostly the integration
+with other tools) manually before merging.
+
+### Review periodic jobs
+
+We run periodic jobs (see also Automated testing section of this documentation).
+Those should be monitored for failures.
+
+If a failure happen in periodics, something terribly wrong must have happened
+(or github is failing at the creation of a kind cluster). Please monitor those
+failures carefully.
+
+### Introducing new features
+
+When you introduce a new feature, the kured team expects you to have tested
+your change thoroughly. If possible, include all the necessary testing in your change.
+
+If your change involves a user facing change (change in flags of kured for example),
+please include expose your new feature in our default manifest (`kured-ds.yaml`),
+as a comment.
+
+Do not update the helm chart directly.
+Helm charts and our release manifests (see below) are our stable interfaces.
+Any user facing changes will therefore have to wait for a while before being
+exposed to our users.
+
+This also means that when you expose a new feature, you should create another PR
+for your changes in `charts/` to make your feature available for our next kured version.
+In this change, you can directly bump the appVersion to the next minor version.
+(for example, if current appVersion is 1.6.x, make sure you update your appVersion
+to 1.7.0). It allows us to have an easy view of what we land each release.
+
+Do not hesitate to increase the test coverage for your feature, whether it's unit
+testing to full functional testing (even using helm charts)
+
+### Increasing test coverage
+
+We are welcoming any change to increase our test coverage.
+See also our github issues for the label `testing`.
+
+### Updating helm charts
+
+Helm charts are continuously published. Any change in `charts/` will be immediately
+pushed in production.
+
+## Automated testing
+
+Our CI is covered by github actions.
+You can see their contents in .github/workflows.
+
+We currently run:
+- go tests and lint
+- shellcheck
+- a check for dead links in our docs
+- a security check against our base image (alpine)
+- a deep functional test using our manifests on all supported k8s versions
+- basic deployment using our helm chart on any chart change
+
+Changes in helm charts are not functionally tested on PRs. We assume that
+the PRs to implement the feature are properly tested by our users and
+contributors before merge.
+
+To test your code manually, follow the section Manual testing.
+
+## Manual (release) testing
 
 Before `kured` is released, we want to make sure it still works fine on the
 previous, current and next minor version of Kubernetes (with respect to the
-embedded `client-go` & `kubectl`). For local testing e.g. `minikube` or
-`kind` can be sufficient.
+`client-go` & `kubectl` dependencies in use). For local testing e.g.
+`minikube` or `kind` can be sufficient. This will allow you to catch issues
+that might not have been tested in our CI, like integration with other tools,
+or your specific use case.
 
 Deploy kured in your test scenario, make sure you pass the right `image`,
 update the e.g. `period` and `reboot-days` options, so you get immediate
@@ -42,7 +119,11 @@ results, if you login to a node and run:
 sudo touch /var/run/reboot-required
 ```
 
-### Testing with `minikube`
+### Example of golang testing
+
+Please run `make test`. You should have golint installed.
+
+### Example of testing with `minikube`
 
 A test-run with `minikube` could look like this:
 
@@ -82,16 +163,16 @@ If all the tests ran well, kured maintainers can reach out to the Weaveworks
 team to get an upcoming `kured` release tested in the Dev environment for
 real life testing.
 
-### Testing with `kind`
+### Example of testing with `kind`
 
 A test-run with `kind` could look like this:
 
 ```console
 # create kind cluster
-kind create cluster --config .github/kind-cluster.yaml
+kind create cluster --config .github/kind-cluster-<k8s-version>.yaml
 
 # create reboot required files on pre-defined kind nodes
-./tests/create-reboot-sentinels.sh
+./tests/kind/create-reboot-sentinels.sh
 
 # check if reboot is working fine
 ./tests/kind/follow-coordinated-reboot.sh
@@ -101,27 +182,20 @@ kind create cluster --config .github/kind-cluster.yaml
 ## Publishing a new kured release
 
 ### Prepare Documentation
+
 Check that `README.md` has an updated compatibility matrix and that the
 url in the `kubectl` incantation (under "Installation") is updated to the
 new version you want to release.
 
-### Create a tag on the repo and publish the image
+### Create a tag on the repo
 
 Before going further, we should freeze the code for a release, by
-tagging the code, and publishing its immutable artifact: the kured
-docker image.
+tagging the code. The Github-Action should start a new job and push
+the new image to the registry.
 
-```sh
-make DH_ORG="weaveworks" VERSION="1.3.0" image
-```
-
-Then docker push the image. In the future, that might be automatically
-done when creating a tag on the repository, with the help of github
-actions.
 
 ### Create the combined manifest
 
-
 Now create the `kured-<release>-dockerhub.yaml` for e.g. `1.3.0`:
 
 ```sh
@@ -131,6 +205,7 @@ make DH_ORG="weaveworks" VERSION="${VERSION}" manifest
 cat kured-rbac.yaml > "$MANIFEST"
 cat kured-ds.yaml >> "$MANIFEST"
 ```
+
 ### Publish release artifacts
 
 Now you can head to the Github UI, use the version number as tag and upload the
@@ -155,3 +230,6 @@ A change in the helm chart requires a bump of the `version`
 in `charts/kured/Chart.yaml` (following the versioning rules).
 Update it, and issue a PR. Upon merge, that PR will automatically
 publish the chart to the gh-pages branch.
+
+When there are open helm-chart PRs which are on hold until the helm-chart has been updated
+with the new kured version, they can be merged now (unless a rebase is needed from the contributor).

MAINTAINERS

@@ -0,0 +1,5 @@
+Christian Kotzbauer <christian.kotzbauer@gmail.com> (@ckotzbauer)
+Daniel Holbach <daniel@weave.works> (@dholbach)
+Hidde Beydals <hidde@weave.works> (@hiddeco)
+Jean-Phillipe Evrard <jean-philippe.evrard@suse.com> (@evrardjp)
+Jack Francis <jackfrancis@gmail.com> (@jackfrancis)

Makefile

@@ -1,5 +1,5 @@
 .DEFAULT: all
-.PHONY: all clean image publish-image minikube-publish manifest helm-chart
+.PHONY: all clean image publish-image minikube-publish manifest helm-chart test tests
 
 DH_ORG=weaveworks
 VERSION=$(shell git symbolic-ref --short HEAD)-$(shell git rev-parse --short HEAD)
@@ -24,12 +24,14 @@ build/.image.done: cmd/kured/Dockerfile cmd/kured/kured
 	cp $^ build
 	$(SUDO) docker build -t docker.io/$(DH_ORG)/kured -f build/Dockerfile ./build
 	$(SUDO) docker tag docker.io/$(DH_ORG)/kured docker.io/$(DH_ORG)/kured:$(VERSION)
+	$(SUDO) docker tag docker.io/$(DH_ORG)/kured ghcr.io/$(DH_ORG)/kured:$(VERSION)
 	touch $@
 
 image: build/.image.done
 
 publish-image: image
 	$(SUDO) docker push docker.io/$(DH_ORG)/kured:$(VERSION)
+	$(SUDO) docker push ghcr.io/$(DH_ORG)/kured:$(VERSION)
 
 minikube-publish: image
 	$(SUDO) docker save docker.io/$(DH_ORG)/kured | (eval $$(minikube docker-env) && docker load)
@@ -40,6 +42,14 @@ manifest:
 
 helm-chart:
 	sed -i "s#repository:.*/kured#repository: $(DH_ORG)/kured#g" charts/kured/values.yaml
-	sed -i "s#tag:.*#tag: $(VERSION)#g" charts/kured/values.yaml
 	sed -i "s#appVersion:.*#appVersion: \"$(VERSION)\"#g" charts/kured/Chart.yaml
+	sed -i "s#\`[0-9]*\.[0-9]*\.[0-9]*\`#\`$(VERSION)\`#g" charts/kured/README.md
 	echo "Please bump version in charts/kured/Chart.yaml"
+
+test: tests
+	echo "Running go tests"
+	go test ./...
+	echo "Running golint on pkg"
+	golint ./pkg/...
+	echo "Running golint on cmd"
+	golint ./cmd/...

README.md

@@ -1,27 +1,29 @@
 
 # kured - Kubernetes Reboot Daemon
 
-<img src="https://github.com/weaveworks/kured/raw/master/img/logo.png" align="right"/>
+<img src="https://github.com/weaveworks/kured/raw/main/img/logo.png" align="right"/>
 
-* [Introduction](#introduction)
-* [Kubernetes & OS Compatibility](#kubernetes-&-os-compatibility)
-* [Installation](#installation)
-* [Configuration](#configuration)
-  * [Reboot Sentinel File & Period](#reboot-sentinel-file-&-period)
-  * [Setting a schedule](#setting-a-schedule)
-  * [Blocking Reboots via Alerts](#blocking-reboots-via-alerts)
-  * [Blocking Reboots via Pods](#blocking-reboots-via-pods)
-  * [Prometheus Metrics](#prometheus-metrics)
-  * [Slack Notifications](#slack-notifications)
-  * [Overriding Lock Configuration](#overriding-lock-configuration)
-* [Operation](#operation)
-  * [Testing](#testing)
-  * [Disabling Reboots](#disabling-reboots)
-  * [Manual Unlock](#manual-unlock)
-  * [Automatic Unlock](#automatic-unlock)
-* [Building](#building)
-* [Frequently Asked/Anticipated Questions](#frequently-askedanticipated-questions)
-* [Getting Help](#getting-help)
+- [Introduction](#introduction)
+- [Kubernetes & OS Compatibility](#kubernetes--os-compatibility)
+- [Installation](#installation)
+- [Configuration](#configuration)
+  - [Reboot Sentinel File & Period](#reboot-sentinel-file--period)
+  - [Setting a schedule](#setting-a-schedule)
+  - [Blocking Reboots via Alerts](#blocking-reboots-via-alerts)
+  - [Blocking Reboots via Pods](#blocking-reboots-via-pods)
+  - [Prometheus Metrics](#prometheus-metrics)
+  - [Notifications](#notifications)
+  - [Overriding Lock Configuration](#overriding-lock-configuration)
+- [Operation](#operation)
+  - [Testing](#testing)
+  - [Disabling Reboots](#disabling-reboots)
+  - [Manual Unlock](#manual-unlock)
+  - [Automatic Unlock](#automatic-unlock)
+  - [Delaying Lock Release](#delaying-lock-release)
+- [Building](#building)
+- [Frequently Asked/Anticipated Questions](#frequently-askedanticipated-questions)
+  - [Why is there no `latest` tag on Docker Hub?](#why-is-there-no-latest-tag-on-docker-hub)
+- [Getting Help](#getting-help)
 
 ## Introduction
 
@@ -29,7 +31,8 @@ Kured (KUbernetes REboot Daemon) is a Kubernetes daemonset that
 performs safe automatic node reboots when the need to do so is
 indicated by the package management system of the underlying OS.
 
-* Watches for the presence of a reboot sentinel e.g. `/var/run/reboot-required`
+* Watches for the presence of a reboot sentinel file e.g. `/var/run/reboot-required`
+  or the successful run of a sentinel command.
 * Utilises a lock in the API server to ensure only one node reboots at
   a time
 * Optionally defers reboots in the presence of active Prometheus alerts or selected pods
@@ -43,16 +46,19 @@ maintaining the lock and draining worker nodes. Kubernetes aims to provide
 forwards and backwards compatibility of one minor version between client and
 server:
 
-| kured  | kubectl | k8s.io/client-go | k8s.io/apimachinery | expected kubernetes compatibility |
-|--------|---------|------------------|---------------------|-----------------------------------|
-| master | 1.19.4  | v0.19.4          | v0.19.4             | 1.18.x, 1.19.x, 1.20.x            |
-| 1.6.0  | 1.19.4  | v0.19.4          | v0.19.4             | 1.18.x, 1.19.x, 1.20.x            |
-| 1.5.1  | 1.18.8  | v0.18.8          | v0.18.8             | 1.17.x, 1.18.x, 1.19.x            |
-| 1.4.4  | 1.17.7  | v0.17.0          | v0.17.0             | 1.16.x, 1.17.x, 1.18.x            |
-| 1.3.0  | 1.15.10 | v12.0.0          | release-1.15        | 1.15.x, 1.16.x, 1.17.x            |
-| 1.2.0  | 1.13.6  | v10.0.0          | release-1.13        | 1.12.x, 1.13.x, 1.14.x            |
-| 1.1.0  | 1.12.1  | v9.0.0           | release-1.12        | 1.11.x, 1.12.x, 1.13.x            |
-| 1.0.0  | 1.7.6   | v4.0.0           | release-1.7         | 1.6.x, 1.7.x, 1.8.x               |
+| kured | kubectl | k8s.io/client-go | k8s.io/apimachinery | expected kubernetes compatibility |
+|-------|---------|------------------|---------------------|-----------------------------------|
+| main  | 1.22.4  | v0.22.4          | v0.22.4             | 1.21.x, 1.22.x, 1.23.x            |
+| 1.9.1 | 1.22.4  | v0.22.4          | v0.22.4             | 1.21.x, 1.22.x, 1.23.x            |
+| 1.8.1 | 1.21.4  | v0.21.4          | v0.21.4             | 1.20.x, 1.21.x, 1.22.x            |
+| 1.7.0 | 1.20.5  | v0.20.5          | v0.20.5             | 1.19.x, 1.20.x, 1.21.x            |
+| 1.6.1 | 1.19.4  | v0.19.4          | v0.19.4             | 1.18.x, 1.19.x, 1.20.x            |
+| 1.5.1 | 1.18.8  | v0.18.8          | v0.18.8             | 1.17.x, 1.18.x, 1.19.x            |
+| 1.4.4 | 1.17.7  | v0.17.0          | v0.17.0             | 1.16.x, 1.17.x, 1.18.x            |
+| 1.3.0 | 1.15.10 | v12.0.0          | release-1.15        | 1.15.x, 1.16.x, 1.17.x            |
+| 1.2.0 | 1.13.6  | v10.0.0          | release-1.13        | 1.12.x, 1.13.x, 1.14.x            |
+| 1.1.0 | 1.12.1  | v9.0.0           | release-1.12        | 1.11.x, 1.12.x, 1.13.x            |
+| 1.0.0 | 1.7.6   | v4.0.0           | release-1.7         | 1.6.x, 1.7.x, 1.8.x               |
 
 See the [release notes](https://github.com/weaveworks/kured/releases)
 for specific version compatibility information, including which
@@ -80,25 +86,37 @@ The following arguments can be passed to kured via the daemonset pod template:
 
 ```console
 Flags:
-      --lock-ttl time                       force clean annotation after this ammount of time (default 0, disabled)
       --alert-filter-regexp regexp.Regexp   alert names to ignore when checking for active alerts
+      --alert-firing-only bool              only consider firing alerts when checking for active alerts
       --blocking-pod-selector stringArray   label selector identifying pods whose presence should prevent reboots
+      --drain-grace-period int              time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1)
+      --skip-wait-for-delete-timeout int    when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0)
       --ds-name string                      name of daemonset on which to place lock (default "kured")
       --ds-namespace string                 namespace containing daemonset on which to place lock (default "kube-system")
-      --end-time string                     only reboot before this time of day (default "23:59")
+      --end-time string                     schedule reboot only before this time of day (default "23:59:59")
+      --force-reboot bool                   force a reboot even if the drain is still running (default: false)
+      --drain-timeout duration              timeout after which the drain is aborted (default: 0, infinite time)
   -h, --help                                help for kured
       --lock-annotation string              annotation in which to record locking node (default "weave.works/kured-node-lock")
-      --period duration                     reboot check period (default 1h0m0s)
-      --prometheus-url string               Prometheus instance to probe for active alerts
-      --reboot-days strings                 only reboot on these days (default [su,mo,tu,we,th,fr,sa])
-      --reboot-sentinel string              path to file whose existence signals need to reboot (default "/var/run/reboot-required")
-      --slack-channel string                slack channel for reboot notfications
-      --slack-hook-url string               slack hook URL for reboot notfications
-      --slack-username string               slack username for reboot notfications (default "kured")
+      --lock-release-delay duration         hold lock after reboot by this duration (default: 0, disabled)
+      --lock-ttl duration                   expire lock annotation after this duration (default: 0, disabled)
       --message-template-drain string       message template used to notify about a node being drained (default "Draining node %s")
       --message-template-reboot string      message template used to notify about a node being rebooted (default "Rebooting node %s")
-      --start-time string                   only reboot after this time of day (default "0:00")
-      --time-zone string                    use this timezone to calculate allowed reboot time (default "UTC")
+      --notify-url                          url for reboot notifications (cannot use with --slack-hook-url flags)
+      --period duration                     reboot check period (default 1h0m0s)
+      --prefer-no-schedule-taint string     Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Disabled by default. Set e.g. to "weave.works/kured-node-reboot" to enable tainting.
+      --prometheus-url string               Prometheus instance to probe for active alerts
+      --reboot-command string               command to run when a reboot is required by the sentinel (default "/sbin/systemctl reboot")
+      --reboot-days strings                 schedule reboot on these days (default [su,mo,tu,we,th,fr,sa])
+      --reboot-delay duration               add a delay after drain finishes but before the reboot command is issued (default 0, no time)
+      --reboot-sentinel string              path to file whose existence signals need to reboot (default "/var/run/reboot-required")
+      --reboot-sentinel-command string      command for which a successful run signals need to reboot (default ""). If non-empty, sentinel file will be ignored.
+      --slack-channel string                slack channel for reboot notfications
+      --slack-hook-url string               slack hook URL for reboot notfications [deprecated in favor of --notify-url]
+      --slack-username string               slack username for reboot notfications (default "kured")
+      --start-time string                   schedule reboot only after this time of day (default "0:00")
+      --time-zone string                    use this timezone for schedule inputs (default "UTC")
+      --log-format string                   log format specified as text or json, defaults to "text"
 ```
 
 ### Reboot Sentinel File & Period
@@ -109,6 +127,10 @@ values with `--reboot-sentinel` and `--period`. Each replica of the
 daemon uses a random offset derived from the period on startup so that
 nodes don't all contend for the lock simultaneously.
 
+Alternatively, a reboot sentinel command can be used. If a reboot
+sentinel command is used, the reboot sentinel file presence will be
+ignored.
+
 ### Setting a schedule
 
 By default, kured will reboot any time it detects the sentinel, but this
@@ -149,6 +171,11 @@ will block reboots, however you can ignore specific alerts:
 --alert-filter-regexp=^(RebootRequired|AnotherBenignAlert|...$
 ```
 
+You can also only block reboots for firing alerts:
+```console
+--alert-firing-only=true
+```
+
 See the section on Prometheus metrics for an important application of this
 filter.
 
@@ -211,21 +238,34 @@ If you choose to employ such an alert and have configured kured to
 probe for active alerts before rebooting, be sure to specify
 `--alert-filter-regexp=^RebootRequired$` to avoid deadlock!
 
-### Slack Notifications
+### Notifications
 
-If you specify a Slack hook via `--slack-hook-url`, kured will notify
-you immediately prior to rebooting a node:
+When you specify a formatted URL using `--notify-url`, kured will notify
+about draining and rebooting nodes across a list of technologies.
 
 ![Notification](img/slack-notification.png)
 
-We recommend setting `--slack-username` to be the name of the
-environment, e.g. `dev` or `prod`.
-
 Alternatively you can use the `--message-template-drain` and `--message-template-reboot` to customize the text of the message, e.g.
-```
+
+```cli
 --message-template-drain="Draining node %s part of *my-cluster* in region *xyz*"
 ```
 
+Here is the syntax:
+
+- slack:           `slack://tokenA/tokenB/tokenC`
+(`--slack-hook-url` is deprecated but possible to use)
+
+- rocketchat:      `rocketchat://[username@]rocketchat-host/token[/channel|@recipient]`
+
+- teams:           `teams://tName/token-a/token-b/token-c`
+
+   > **Attention** as the [format of the url has changed](https://github.com/containrrr/shoutrrr/issues/138) you also have to specify a `tName`
+
+- Email:           `smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]`
+
+More details here: [containrrr.dev/shoutrrr/v0.4/services/overview](https://containrrr.dev/shoutrrr/v0.4/services/overview)
+
 ### Overriding Lock Configuration
 
 The `--ds-name` and `--ds-namespace` arguments should match the name and
@@ -282,6 +322,11 @@ which holds lock might be killed thus annotation will stay there for ever.
 
 Using `--lock-ttl=30m` will allow other nodes to take over if TTL has expired (in this case 30min) and continue reboot process.
 
+### Delaying Lock Release
+
+
+Using `--lock-release-delay=30m` will cause nodes to hold the lock for the specified time frame (in this case 30min) before it is released and the reboot process continues. This can be used to throttle reboots across the cluster.
+
 ## Building
 
 Kured now uses [Go
@@ -314,7 +359,7 @@ our [development][development] docs.
 
 Use of `latest` for production deployments is bad practice - see
 [here](https://kubernetes.io/docs/concepts/configuration/overview) for
-details. The manifest on `master` refers to `latest` for local
+details. The manifest on `main` refers to `latest` for local
 development testing with minikube only; for production use choose a
 versioned manifest from the [release page](https://github.com/weaveworks/kured/releases/).
 
@@ -328,4 +373,6 @@ If you have any questions about, feedback for or problems with `kured`:
 * Join us in [our monthly meeting](https://docs.google.com/document/d/1bsHTjHhqaaZ7yJnXF6W8c89UB_yn-OoSZEmDnIP34n8/edit#),
   every fourth Wednesday of the month at 16:00 UTC.
 
+We follow the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
+
 Your feedback is always welcome!

charts/kured/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
-appVersion: "1.5.1"
+appVersion: "1.9.1"
 description: A Helm chart for kured
 name: kured
-version: 2.2.3
+version: 2.11.2
 home: https://github.com/weaveworks/kured
 maintainers:
   - name: ckotzbauer
@@ -11,4 +11,4 @@ maintainers:
     email: david@davidkarlsen.com
 sources:
   - https://github.com/weaveworks/kured
-icon: https://raw.githubusercontent.com/weaveworks/kured/master/img/logo.png
+icon: https://raw.githubusercontent.com/weaveworks/kured/main/img/logo.png

charts/kured/README.md

@@ -36,29 +36,44 @@ The following changes have been made compared to the stable chart:
 | Config                  | Description                                                                 | Default                    |
 | ------                  | -----------                                                                 | -------                    |
 | `image.repository`      | Image repository                                                            | `weaveworks/kured` |
-| `image.tag`             | Image tag                                                                   | `1.5.1`                    |
+| `image.tag`             | Image tag                                                                   | `1.9.1`                    |
 | `image.pullPolicy`      | Image pull policy                                                           | `IfNotPresent`             |
 | `image.pullSecrets`     | Image pull secrets                                                          | `[]`                       |
-| `updateStrategy`        | Daemonset update strategy                                                   | `OnDelete`                 |
+| `updateStrategy`        | Daemonset update strategy                                                   | `RollingUpdate`            |
+| `maxUnavailable`        | The max pods unavailable during a rolling update                            | `1`                        |
 | `podAnnotations`        | Annotations to apply to pods (eg to add Prometheus annotations)             | `{}`                       |
+| `dsAnnotations`         | Annotations to apply to the kured DaemonSet                                 | `{}`                       |
 | `extraArgs`             | Extra arguments to pass to `/usr/bin/kured`. See below.                     | `{}`                       |
 | `extraEnvVars`          | Array of environment variables to pass to the daemonset.                    | `{}`                       |
 | `configuration.lockTtl` | cli-parameter `--lock-ttl`                                                  | `0`                       |
+| `configuration.lockReleaseDelay` | cli-parameter `--lock-release-delay`                               | `0`                       |
 | `configuration.alertFilterRegexp` | cli-parameter `--alert-filter-regexp`                             | `""`                       |
+| `configuration.alertFiringOnly` | cli-parameter `--alert-firing-only`                                 | `false`                   |
 | `configuration.blockingPodSelector` | Array of selectors for multiple cli-parameters `--blocking-pod-selector` | `[]`             |
 | `configuration.endTime` | cli-parameter `--end-time`                                                  | `""`                      |
 | `configuration.lockAnnotation` | cli-parameter `--lock-annotation`                                    | `""`                      |
 | `configuration.period` | cli-parameter `--period`                                                     | `""`                      |
+| `configuration.forceReboot` | cli-parameter `--force-reboot`                                          | `false`                   |
+| `configuration.drainGracePeriod` | cli-parameter `--drain-grace-period`                               | `""`                      |
+| `configuration.drainTimeout` | cli-parameter `--drain-timeout`                                        | `""`                      |
+| `configuration.skipWaitForDeleteTimeout` | cli-parameter `--skip-wait-for-delete-timeout`             | `""`                      |
 | `configuration.prometheusUrl` | cli-parameter `--prometheus-url`                                      | `""`                      |
 | `configuration.rebootDays` | Array of days for multiple cli-parameters `--reboot-days`                | `[]`                      |
 | `configuration.rebootSentinel` | cli-parameter `--reboot-sentinel`                                    | `""`                      |
+| `configuration.rebootSentinelCommand` | cli-parameter `--reboot-sentinel-command`                     | `""`                      |
+| `configuration.rebootCommand` | cli-parameter `--reboot-command`                                      | `""`                      |
+| `configuration.rebootDelay` | cli-parameter `--reboot-delay`                                          | `""`                      |
 | `configuration.slackChannel` | cli-parameter `--slack-channel`                                        | `""`                      |
 | `configuration.slackHookUrl` | cli-parameter `--slack-hook-url`                                       | `""`                      |
 | `configuration.slackUsername` | cli-parameter `--slack-username`                                      | `""`                      |
+| `configuration.notifyUrl` | cli-parameter `--notify-url`                                              | `""`                      |
 | `configuration.messageTemplateDrain` | cli-parameter `--message-template-drain`                       | `""`                      |
 | `configuration.messageTemplateReboot` | cli-parameter `--message-template-reboot`                     | `""`                      |
 | `configuration.startTime` | cli-parameter `--start-time`                                              | `""`                      |
 | `configuration.timeZone` | cli-parameter `--time-zone`                                                | `""`                      |
+| `configuration.annotateNodes` | cli-parameter `--annotate-nodes`                                      | `false`                   |
+| `configuration.logFormat` | cli-parameter `--log-format`                                              | `"text"`                  |
+| `configuration.preferNoScheduleTaint` | Taint name applied during pending node reboot                 | `""`                   |
 | `rbac.create`           | Create RBAC roles                                                           | `true`                     |
 | `serviceAccount.create` | Create a service account                                                    | `true`                     |
 | `serviceAccount.name`   | Service account name to create (or use if `serviceAccount.create` is false) | (chart fullname)           |
@@ -70,13 +85,16 @@ The following changes have been made compared to the stable chart:
 | `metrics.interval`      | Interval prometheus should scrape the endpoint                              | `60s`                   |
 | `metrics.scrapeTimeout` | A custom scrapeTimeout for prometheus                                       | `""`                    |
 | `service.create`        | Create a Service for the metrics endpoint                                   | `false`                    |
+| `service.name  `        | Service name for the metrics endpoint                                       | `""`                       |
 | `service.port`          | Port of the service to expose                                               | `8080`                     |
 | `service.annotations`   | Annotations to apply to the service (eg to add Prometheus annotations)      | `{}`                       |
+| `podLabels`             | Additional labels for pods (e.g. CostCenter=IT)                             | `{}`                       |
 | `priorityClassName`     | Priority Class to be used by the pods                                       | `""`                       |
 | `tolerations`           | Tolerations to apply to the daemonset (eg to allow running on master)       | `[{"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}]`|
 | `affinity`              | Affinity for the daemonset (ie, restrict which nodes kured runs on)         | `{}`                       |
 | `nodeSelector`          | Node Selector for the daemonset (ie, restrict which nodes kured runs on)    | `{}`                       |
-
+| `volumeMounts`          | Maps of volumes mount to mount                                              | `{}`                       |
+| `volumes`               | Maps of volumes to mount                                                    | `{}`                       |
 See https://github.com/weaveworks/kured#configuration for values (not contained in the `configuration` object) for `extraArgs`. Note that
 ```yaml
 extraArgs:

charts/kured/ci/prometheus-values.yaml

@@ -0,0 +1,13 @@
+# This is tested twice:
+# Basic install test with chart-testing (on charts PRs)
+# Functional testing in PRs (other PRs)
+
+service:
+  create: true
+  name: kured-prometheus-endpoint
+  port: 8080
+  type: NodePort
+  nodePort: 30000
+
+# Do not override the configuration: period in this, so that
+# We can test prometheus exposed metrics without rebooting.

charts/kured/templates/daemonset.yaml

@@ -5,9 +5,19 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "kured.labels" . | nindent 4 }}
+  {{- if .Values.dsAnnotations }}
+  annotations:
+  {{- range $key, $value := .Values.dsAnnotations }}
+    {{ $key }}: {{ $value | quote }}
+  {{- end }}
+  {{- end }}
 spec:
   updateStrategy:
     type: {{ .Values.updateStrategy }}
+{{- if eq .Values.updateStrategy "RollingUpdate"}}
+    rollingUpdate:
+      maxUnavailable: {{ .Values.maxUnavailable }}
+{{- end}}
   selector:
     matchLabels:
       {{- include "kured.matchLabels" . | nindent 6 }}
@@ -15,6 +25,9 @@ spec:
     metadata:
       labels:
         {{- include "kured.labels" . | nindent 8 }}
+        {{- if .Values.podLabels }}
+        {{- toYaml .Values.podLabels | nindent 8 }}
+        {{- end }}
       {{- if .Values.podAnnotations }}
       annotations:
       {{- range $key, $value := .Values.podAnnotations }}
@@ -34,7 +47,7 @@ spec:
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           securityContext:
             privileged: true # Give permission to nsenter /proc/1/ns/mnt
@@ -48,9 +61,15 @@ spec:
           {{- if .Values.configuration.lockTtl }}
             - --lock-ttl={{ .Values.configuration.lockTtl }}
           {{- end }}
+          {{- if .Values.configuration.lockReleaseDelay }}
+            - --lock-release-delay={{ .Values.configuration.lockReleaseDelay }}
+          {{- end }}
           {{- if .Values.configuration.alertFilterRegexp }}
             - --alert-filter-regexp={{ .Values.configuration.alertFilterRegexp }}
           {{- end }}
+          {{- if .Values.configuration.alertFiringOnly }}
+            - --alert-firing-only={{ .Values.configuration.alertFiringOnly }}
+          {{- end }}
           {{- range .Values.configuration.blockingPodSelector }}
             - --blocking-pod-selector={{ . }}
           {{- end }}
@@ -63,6 +82,18 @@ spec:
           {{- if .Values.configuration.period }}
             - --period={{ .Values.configuration.period }}
           {{- end }}
+          {{- if .Values.configuration.forceReboot }}
+            - --force-reboot
+          {{- end }}
+          {{- if .Values.configuration.drainGracePeriod }}
+            - --drain-grace-period={{ .Values.configuration.drainGracePeriod }}
+          {{- end }}
+          {{- if .Values.configuration.drainTimeout }}
+            - --drain-timeout={{ .Values.configuration.drainTimeout }}
+          {{- end }}
+          {{- if .Values.configuration.skipWaitForDeleteTimeout }}
+             - --skip-wait-for-delete-timeout={{ .Values.configuration.skipWaitForDeleteTimeout }}
+          {{- end }}
           {{- if .Values.configuration.prometheusUrl }}
             - --prometheus-url={{ .Values.configuration.prometheusUrl }}
           {{- end }}
@@ -72,6 +103,15 @@ spec:
           {{- if .Values.configuration.rebootSentinel }}
             - --reboot-sentinel={{ .Values.configuration.rebootSentinel }}
           {{- end }}
+          {{- if .Values.configuration.rebootSentinelCommand }}
+            - --reboot-sentinel-command={{ .Values.configuration.rebootSentinelCommand }}
+          {{- end }}
+          {{- if .Values.configuration.rebootCommand }}
+            - --reboot-command={{ .Values.configuration.rebootCommand }}
+          {{- end }}
+          {{- if .Values.configuration.rebootDelay }}
+            - --reboot-delay={{ .Values.configuration.rebootDelay }}
+          {{- end }}
           {{- if .Values.configuration.slackChannel }}
             - --slack-channel={{ .Values.configuration.slackChannel }}
           {{- end }}
@@ -81,6 +121,9 @@ spec:
           {{- if .Values.configuration.slackUsername }}
             - --slack-username={{ .Values.configuration.slackUsername }}
           {{- end }}
+          {{- if .Values.configuration.notifyUrl }}
+            - --notify-url={{ .Values.configuration.notifyUrl }}
+          {{- end }}
           {{- if .Values.configuration.messageTemplateDrain }}
             - --message-template-drain={{ .Values.configuration.messageTemplateDrain }}
           {{- end }}
@@ -93,6 +136,15 @@ spec:
           {{- if .Values.configuration.timeZone }}
             - --time-zone={{ .Values.configuration.timeZone }}
           {{- end }}
+          {{- if .Values.configuration.annotateNodes }}
+            - --annotate-nodes={{ .Values.configuration.annotateNodes }}
+          {{- end }}
+          {{- if .Values.configuration.preferNoScheduleTaint }}
+            - --prefer-no-schedule-taint={{ .Values.configuration.preferNoScheduleTaint }}
+          {{- end }}
+          {{- if .Values.configuration.logFormat }}
+            - --log-format={{ .Values.configuration.logFormat }}
+          {{- end }}
           {{- range $key, $value := .Values.extraArgs }}
             {{- if $value }}
             - --{{ $key }}={{ $value }}
@@ -100,6 +152,10 @@ spec:
             - --{{ $key }}
             {{- end }}
           {{- end }}
+{{- if .Values.volumeMounts }}
+          volumeMounts:
+{{- toYaml .Values.volumeMounts  | nindent 12 }}
+{{- end }}
           ports:
             - containerPort: 8080
               name: metrics
@@ -125,3 +181,7 @@ spec:
       affinity:
 {{ toYaml . | indent 8 }}
       {{- end }}
+{{- if .Values.volumes }}
+      volumes:
+{{- toYaml .Values.volumes  | nindent 8 }}
+{{- end }}

charts/kured/templates/service.yaml

@@ -2,7 +2,11 @@
 apiVersion: v1
 kind: Service
 metadata:
+  {{- if .Values.service.name }}
+  name: {{ .Values.service.name }}
+  {{- else }}
   name: {{ template "kured.fullname" . }}
+  {{- end }}
   labels:
     {{- include "kured.labels" . | nindent 4 }}
   {{- if .Values.service.annotations }}
@@ -12,11 +16,14 @@ metadata:
   {{- end }}
   {{- end }}
 spec:
-  type: ClusterIP
+  type: {{ .Values.service.type }}
   ports:
     - name: metrics
       port: {{ .Values.service.port }}
       targetPort: 8080
+      {{- if eq .Values.service.type "NodePort" }}
+      nodePort: {{ .Values.service.nodePort }}
+      {{- end }}
   selector:
     {{- include "kured.matchLabels" . | nindent 4 }}
-{{- end }}
+{{- end }}

charts/kured/values.minikube.yaml

@@ -3,19 +3,29 @@ image:
   tag: latest
 
 configuration:
-  # annotationTtl: 0          # force clean annotation after this ammount of time (default 0, disabled)
+  # annotationTtl: 0          # force clean annotation after this amount of time (default 0, disabled)
   # alertFilterRegexp: ""     # alert names to ignore when checking for active alerts
+  # alertFiringOnly: false     # only consider firing alerts when checking for active alerts
   # blockingPodSelector: []   # label selector identifying pods whose presence should prevent reboots
   # endTime: ""               # only reboot before this time of day (default "23:59")
   # lockAnnotation: ""        # annotation in which to record locking node (default "weave.works/kured-node-lock")
   period: "1m"                # reboot check period (default 1h0m0s)
+  # forceReboot: false        # force a reboot even if the drain fails or times out (default: false)
+  # drainGracePeriod: ""      # time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1)
+  # drainTimeout: ""          # timeout after which the drain is aborted (default: 0, infinite time)
+  # skipWaitForDeleteTimeout: ""  # when time is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0)
   # prometheusUrl: ""         # Prometheus instance to probe for active alerts
   # rebootDays: []            # only reboot on these days (default [su,mo,tu,we,th,fr,sa])
   # rebootSentinel: ""        # path to file whose existence signals need to reboot (default "/var/run/reboot-required")
+  # rebootSentinelCommand: ""  # command for which a successful run signals need to reboot (default ""). If non-empty, sentinel file will be ignored.
   # slackChannel: ""          # slack channel for reboot notfications
   # slackHookUrl: ""          # slack hook URL for reboot notfications
   # slackUsername: ""         # slack username for reboot notfications (default "kured")
+  # notifyUrl: ""             # notification URL with the syntax as follows: https://containrrr.dev/shoutrrr/services/overview/
   # messageTemplateDrain: ""  # slack message template when notifying about a node being drained (default "Draining node %s")
   # messageTemplateReboot: "" # slack message template when notifying about a node being rebooted (default "Rebooted node %s")
   # startTime: ""             # only reboot after this time of day (default "0:00")
   # timeZone: ""              # time-zone to use (valid zones from "time" golang package)
+  # annotateNodes: false      # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations
+  # lockReleaseDelay: "5m"    # hold lock after reboot by this amount of time (default 0, disabled)
+  # logFormat: "text"         # log format specified as text or json, defaults to text

charts/kured/values.yaml

@@ -1,12 +1,15 @@
 image:
   repository: weaveworks/kured
-  tag: 1.5.1
+  tag: ""  # will default to the appVersion in Chart.yaml
   pullPolicy: IfNotPresent
   pullSecrets: []
 
-updateStrategy: OnDelete
+updateStrategy: RollingUpdate
+#  requires RollingUpdate updateStrategy
+maxUnavailable: 1
 
 podAnnotations: {}
+dsAnnotations: {}
 
 extraArgs: {}
 
@@ -20,22 +23,35 @@ extraEnvVars:
 #    value: 123
 
 configuration:
-  lockTtl: 0                 # force clean annotation after this ammount of time (default 0, disabled)
+  lockTtl: 0                 # force clean annotation after this amount of time (default 0, disabled)
   alertFilterRegexp: ""      # alert names to ignore when checking for active alerts
+  alertFiringOnly: false     # only consider firing alerts when checking for active alerts
   blockingPodSelector: []    # label selector identifying pods whose presence should prevent reboots
   endTime: ""                # only reboot before this time of day (default "23:59")
   lockAnnotation: ""         # annotation in which to record locking node (default "weave.works/kured-node-lock")
   period: ""                 # reboot check period (default 1h0m0s)
+  forceReboot: false         # force a reboot even if the drain fails or times out (default: false)
+  drainGracePeriod: ""       # time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1)
+  drainTimeout: ""           # timeout after which the drain is aborted (default: 0, infinite time)
+  skipWaitForDeleteTimeout: ""  # when time is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0)
   prometheusUrl: ""          # Prometheus instance to probe for active alerts
   rebootDays: []             # only reboot on these days (default [su,mo,tu,we,th,fr,sa])
   rebootSentinel: ""         # path to file whose existence signals need to reboot (default "/var/run/reboot-required")
+  rebootSentinelCommand: ""  # command for which a successful run signals need to reboot (default ""). If non-empty, sentinel file will be ignored.
+  rebootCommand: "/bin/systemctl reboot"  # command to run when a reboot is required by the sentinel
+  rebootDelay: ""            # add a delay after drain finishes but before the reboot command is issued
   slackChannel: ""           # slack channel for reboot notfications
   slackHookUrl: ""           # slack hook URL for reboot notfications
   slackUsername: ""          # slack username for reboot notfications (default "kured")
+  notifyUrl: ""              # notification URL with the syntax as follows: https://containrrr.dev/shoutrrr/services/overview/
   messageTemplateDrain: ""   # slack message template when notifying about a node being drained (default "Draining node %s")
   messageTemplateReboot: ""  # slack message template when notifying about a node being rebooted (default "Rebooted node %s")
   startTime: ""              # only reboot after this time of day (default "0:00")
   timeZone: ""               # time-zone to use (valid zones from "time" golang package)
+  annotateNodes: false       # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations
+  lockReleaseDelay: 0        # hold lock after reboot by this amount of time (default 0, disabled)
+  preferNoScheduleTaint: ""  # Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Disabled by default. Set e.g. to "weave.works/kured-node-reboot" to enable tainting.
+  logFormat: "text"          # log format specified as text or json, defaults to text
 
 rbac:
   create: true
@@ -60,6 +76,10 @@ service:
   create: false
   port: 8080
   annotations: {}
+  name: ""
+  type: ClusterIP
+
+podLabels: {}
 
 priorityClassName: ""
 
@@ -70,3 +90,7 @@ tolerations:
 affinity: {}
 
 nodeSelector: {}
+
+volumeMounts: []
+
+volumes: []

cmd/kured/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.12
+FROM alpine:3.15.0
 RUN apk update --no-cache && apk upgrade --no-cache && apk add --no-cache ca-certificates tzdata
 COPY ./kured /usr/bin/kured
 ENTRYPOINT ["/usr/bin/kured"]

cmd/kured/Dockerfile.multi

@@ -0,0 +1,19 @@
+FROM --platform=$BUILDPLATFORM golang:bullseye AS build
+
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+ENV GOOS=$TARGETOS
+ENV GOARCH=$TARGETARCH
+ENV GOVARIANT=$TARGETVARIANT
+
+WORKDIR /src
+COPY . .
+RUN go list -f '{{join .Deps "\n"}}' ./cmd/kured | grep -v /vendor/ | xargs go list -f '{{if not .Standard}}{{ $dep := . }}{{range .GoFiles}}{{$dep.Dir}}/{{.}} {{end}}{{end}}'
+RUN CGO_ENABLED=0 go build -o cmd/kured/kured cmd/kured/*.go
+
+FROM --platform=$TARGETPLATFORM alpine:3.15 as bin
+RUN apk update --no-cache && apk upgrade --no-cache && apk add --no-cache ca-certificates tzdata
+COPY --from=build /src/cmd/kured/kured /usr/bin/kured
+ENTRYPOINT ["/usr/bin/kured"]

cmd/kured/main.go

@@ -2,28 +2,38 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"regexp"
+	"strings"
 	"time"
 
+	papi "github.com/prometheus/client_golang/api"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	kubectldrain "k8s.io/kubectl/pkg/drain"
 
+	"github.com/google/shlex"
+
+	shoutrrr "github.com/containrrr/shoutrrr"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/weaveworks/kured/pkg/alerts"
 	"github.com/weaveworks/kured/pkg/daemonsetlock"
 	"github.com/weaveworks/kured/pkg/delaytick"
-	"github.com/weaveworks/kured/pkg/notifications/slack"
+	"github.com/weaveworks/kured/pkg/taints"
 	"github.com/weaveworks/kured/pkg/timewindow"
 )
 
@@ -31,25 +41,39 @@ var (
 	version = "unreleased"
 
 	// Command line flags
-	period                 time.Duration
-	dsNamespace            string
-	dsName                 string
-	lockAnnotation         string
-	lockTTL                time.Duration
-	prometheusURL          string
-	alertFilter            *regexp.Regexp
-	rebootSentinel         string
-	slackHookURL           string
-	slackUsername          string
-	slackChannel           string
-	messageTemplateDrain   string
-	messageTemplateReboot  string
-	podSelectors           []string
+	forceReboot                     bool
+	drainTimeout                    time.Duration
+	rebootDelay                     time.Duration
+	period                          time.Duration
+	drainGracePeriod                int
+	skipWaitForDeleteTimeoutSeconds int
+	dsNamespace                     string
+	dsName                          string
+	lockAnnotation                  string
+	lockTTL                         time.Duration
+	lockReleaseDelay                time.Duration
+	prometheusURL                   string
+	preferNoScheduleTaintName       string
+	alertFilter                     *regexp.Regexp
+	alertFiringOnly                 bool
+	rebootSentinelFile              string
+	rebootSentinelCommand           string
+	notifyURL                       string
+	slackHookURL                    string
+	slackUsername                   string
+	slackChannel                    string
+	messageTemplateDrain            string
+	messageTemplateReboot           string
+	podSelectors                    []string
+	rebootCommand                   string
+	logFormat                       string
+	nodeID                          string
 
-	rebootDays  []string
-	rebootStart string
-	rebootEnd   string
-	timezone    string
+	rebootDays    []string
+	rebootStart   string
+	rebootEnd     string
+	timezone      string
+	annotateNodes bool
 
 	// Metrics
 	rebootRequiredGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -59,39 +83,85 @@ var (
 	}, []string{"node"})
 )
 
+const (
+	// KuredNodeLockAnnotation is the canonical string value for the kured node-lock annotation
+	KuredNodeLockAnnotation string = "weave.works/kured-node-lock"
+	// KuredRebootInProgressAnnotation is the canonical string value for the kured reboot-in-progress annotation
+	KuredRebootInProgressAnnotation string = "weave.works/kured-reboot-in-progress"
+	// KuredMostRecentRebootNeededAnnotation is the canonical string value for the kured most-recent-reboot-needed annotation
+	KuredMostRecentRebootNeededAnnotation string = "weave.works/kured-most-recent-reboot-needed"
+	// EnvPrefix The environment variable prefix of all environment variables bound to our command line flags.
+	EnvPrefix = "KURED"
+)
+
 func init() {
 	prometheus.MustRegister(rebootRequiredGauge)
 }
 
 func main() {
-	rootCmd := &cobra.Command{
-		Use:   "kured",
-		Short: "Kubernetes Reboot Daemon",
-		Run:   root}
+	cmd := NewRootCommand()
 
+	if err := cmd.Execute(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// NewRootCommand construct the Cobra root command
+func NewRootCommand() *cobra.Command {
+	rootCmd := &cobra.Command{
+		Use:               "kured",
+		Short:             "Kubernetes Reboot Daemon",
+		PersistentPreRunE: bindViper,
+		PreRun:            flagCheck,
+		Run:               root}
+
+	rootCmd.PersistentFlags().StringVar(&nodeID, "node-id", "",
+		"node name kured runs on, should be passed down from spec.nodeName via KURED_NODE_ID environment variable")
+	rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false,
+		"force a reboot even if the drain fails or times out (default: false)")
+	rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1,
+		"time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1)")
+	rootCmd.PersistentFlags().IntVar(&skipWaitForDeleteTimeoutSeconds, "skip-wait-for-delete-timeout", 0,
+		"when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0)")
+	rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 0,
+		"timeout after which the drain is aborted (default: 0, infinite time)")
+	rootCmd.PersistentFlags().DurationVar(&rebootDelay, "reboot-delay", 0,
+		"delay reboot for this duration (default: 0, disabled)")
 	rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60,
-		"reboot check period")
+		"sentinel check period")
 	rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system",
 		"namespace containing daemonset on which to place lock")
 	rootCmd.PersistentFlags().StringVar(&dsName, "ds-name", "kured",
 		"name of daemonset on which to place lock")
-	rootCmd.PersistentFlags().StringVar(&lockAnnotation, "lock-annotation", "weave.works/kured-node-lock",
+	rootCmd.PersistentFlags().StringVar(&lockAnnotation, "lock-annotation", KuredNodeLockAnnotation,
 		"annotation in which to record locking node")
 	rootCmd.PersistentFlags().DurationVar(&lockTTL, "lock-ttl", 0,
 		"expire lock annotation after this duration (default: 0, disabled)")
+	rootCmd.PersistentFlags().DurationVar(&lockReleaseDelay, "lock-release-delay", 0,
+		"delay lock release for this duration (default: 0, disabled)")
 	rootCmd.PersistentFlags().StringVar(&prometheusURL, "prometheus-url", "",
 		"Prometheus instance to probe for active alerts")
 	rootCmd.PersistentFlags().Var(&regexpValue{&alertFilter}, "alert-filter-regexp",
 		"alert names to ignore when checking for active alerts")
-	rootCmd.PersistentFlags().StringVar(&rebootSentinel, "reboot-sentinel", "/var/run/reboot-required",
-		"path to file whose existence signals need to reboot")
+	rootCmd.PersistentFlags().BoolVar(&alertFiringOnly, "alert-firing-only", false,
+		"only consider firing alerts when checking for active alerts (default: false)")
+	rootCmd.PersistentFlags().StringVar(&rebootSentinelFile, "reboot-sentinel", "/var/run/reboot-required",
+		"path to file whose existence triggers the reboot command")
+	rootCmd.PersistentFlags().StringVar(&preferNoScheduleTaintName, "prefer-no-schedule-taint", "",
+		"Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Disabled by default. Set e.g. to \"weave.works/kured-node-reboot\" to enable tainting.")
+	rootCmd.PersistentFlags().StringVar(&rebootSentinelCommand, "reboot-sentinel-command", "",
+		"command for which a zero return code will trigger a reboot command")
+	rootCmd.PersistentFlags().StringVar(&rebootCommand, "reboot-command", "/bin/systemctl reboot",
+		"command to run when a reboot is required")
 
 	rootCmd.PersistentFlags().StringVar(&slackHookURL, "slack-hook-url", "",
-		"slack hook URL for reboot notfications")
+		"slack hook URL for notifications")
 	rootCmd.PersistentFlags().StringVar(&slackUsername, "slack-username", "kured",
-		"slack username for reboot notfications")
+		"slack username for notifications")
 	rootCmd.PersistentFlags().StringVar(&slackChannel, "slack-channel", "",
 		"slack channel for reboot notfications")
+	rootCmd.PersistentFlags().StringVar(&notifyURL, "notify-url", "",
+		"notify URL for reboot notfications")
 	rootCmd.PersistentFlags().StringVar(&messageTemplateDrain, "message-template-drain", "Draining node %s",
 		"message template used to notify about a node being drained")
 	rootCmd.PersistentFlags().StringVar(&messageTemplateReboot, "message-template-reboot", "Rebooting node %s",
@@ -109,15 +179,71 @@ func main() {
 	rootCmd.PersistentFlags().StringVar(&timezone, "time-zone", "UTC",
 		"use this timezone for schedule inputs")
 
-	if err := rootCmd.Execute(); err != nil {
-		log.Fatal(err)
+	rootCmd.PersistentFlags().BoolVar(&annotateNodes, "annotate-nodes", false,
+		"if set, the annotations 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' will be given to nodes undergoing kured reboots")
+
+	rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "text",
+		"use text or json log format")
+
+	return rootCmd
+}
+
+// temporary func that checks for deprecated slack-notification-related flags
+func flagCheck(cmd *cobra.Command, args []string) {
+	if slackHookURL != "" && notifyURL != "" {
+		log.Warnf("Cannot use both --notify-url and --slack-hook-url flags. Kured will use --notify-url flag only...")
 	}
+	if slackHookURL != "" {
+		log.Warnf("Deprecated flag(s). Please use --notify-url flag instead.")
+		trataURL, err := url.Parse(slackHookURL)
+		if err != nil {
+			log.Warnf("slack-hook-url is not properly formatted... no notification will be sent: %v\n", err)
+		}
+		if len(strings.Split(strings.Trim(trataURL.Path, "/services/"), "/")) != 3 {
+			log.Warnf("slack-hook-url is not properly formatted... no notification will be sent: unexpected number of / in URL\n")
+		} else {
+			notifyURL = fmt.Sprintf("slack://%s", strings.Trim(trataURL.Path, "/services/"))
+		}
+	}
+}
+
+// bindViper initializes viper and binds command flags with environment variables
+func bindViper(cmd *cobra.Command, args []string) error {
+	v := viper.New()
+
+	v.SetEnvPrefix(EnvPrefix)
+	v.AutomaticEnv()
+	bindFlags(cmd, v)
+
+	return nil
+}
+
+// bindFlags binds each cobra flag to its associated viper configuration (environment variable)
+func bindFlags(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().VisitAll(func(f *pflag.Flag) {
+		// Environment variables can't have dashes in them, so bind them to their equivalent keys with underscores
+		if strings.Contains(f.Name, "-") {
+			v.BindEnv(f.Name, flagToEnvVar(f.Name))
+		}
+
+		// Apply the viper config value to the flag when the flag is not set and viper has a value
+		if !f.Changed && v.IsSet(f.Name) {
+			val := v.Get(f.Name)
+			log.Infof("Binding %s command flag to environment variable: %s", f.Name, flagToEnvVar(f.Name))
+			cmd.Flags().Set(f.Name, fmt.Sprintf("%v", val))
+		}
+	})
+}
+
+// flagToEnvVar converts command flag name to equivalent environment variable name
+func flagToEnvVar(flag string) string {
+	envVarSuffix := strings.ToUpper(strings.ReplaceAll(flag, "-", "_"))
+	return fmt.Sprintf("%s_%s", EnvPrefix, envVarSuffix)
 }
 
 // newCommand creates a new Command with stdout/stderr wired to our standard logger
 func newCommand(name string, arg ...string) *exec.Cmd {
 	cmd := exec.Command(name, arg...)
-
 	cmd.Stdout = log.NewEntry(log.StandardLogger()).
 		WithField("cmd", cmd.Args[0]).
 		WithField("std", "out").
@@ -131,10 +257,19 @@ func newCommand(name string, arg ...string) *exec.Cmd {
 	return cmd
 }
 
-func sentinelExists() bool {
-	// Relies on hostPID:true and privileged:true to enter host mount space
-	sentinelCmd := newCommand("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "--", "/usr/bin/test", "-f", rebootSentinel)
-	if err := sentinelCmd.Run(); err != nil {
+// buildHostCommand writes a new command to run in the host namespace
+// Rancher based need different pid
+func buildHostCommand(pid int, command []string) []string {
+
+	// From the container, we nsenter into the proper PID to run the hostCommand.
+	// For this, kured daemonset need to be configured with hostPID:true and privileged:true
+	cmd := []string{"/usr/bin/nsenter", fmt.Sprintf("-m/proc/%d/ns/mnt", pid), "--"}
+	cmd = append(cmd, command...)
+	return cmd
+}
+
+func rebootRequired(sentinelCommand []string) bool {
+	if err := newCommand(sentinelCommand[0], sentinelCommand[1:]...).Run(); err != nil {
 		switch err := err.(type) {
 		case *exec.ExitError:
 			// We assume a non-zero exit code means 'reboot not required', but of course
@@ -151,35 +286,56 @@ func sentinelExists() bool {
 	return true
 }
 
-func rebootRequired() bool {
-	if sentinelExists() {
-		log.Infof("Reboot required")
+// RebootBlocker interface should be implemented by types
+// to know if their instantiations should block a reboot
+type RebootBlocker interface {
+	isBlocked() bool
+}
+
+// PrometheusBlockingChecker contains info for connecting
+// to prometheus, and can give info about whether a reboot should be blocked
+type PrometheusBlockingChecker struct {
+	// prometheusClient to make prometheus-go-client and api config available
+	// into the PrometheusBlockingChecker struct
+	promClient *alerts.PromClient
+	// regexp used to get alerts
+	filter *regexp.Regexp
+	// bool to indicate if only firing alerts should be considered
+	firingOnly bool
+}
+
+// KubernetesBlockingChecker contains info for connecting
+// to k8s, and can give info about whether a reboot should be blocked
+type KubernetesBlockingChecker struct {
+	// client used to contact kubernetes API
+	client   *kubernetes.Clientset
+	nodename string
+	// lised used to filter pods (podSelector)
+	filter []string
+}
+
+func (pb PrometheusBlockingChecker) isBlocked() bool {
+
+	alertNames, err := pb.promClient.ActiveAlerts(pb.filter, pb.firingOnly)
+	if err != nil {
+		log.Warnf("Reboot blocked: prometheus query error: %v", err)
+		return true
+	}
+	count := len(alertNames)
+	if count > 10 {
+		alertNames = append(alertNames[:10], "...")
+	}
+	if count > 0 {
+		log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
 		return true
 	}
-	log.Infof("Reboot not required")
 	return false
 }
 
-func rebootBlocked(client *kubernetes.Clientset, nodeID string) bool {
-	if prometheusURL != "" {
-		alertNames, err := alerts.PrometheusActiveAlerts(prometheusURL, alertFilter)
-		if err != nil {
-			log.Warnf("Reboot blocked: prometheus query error: %v", err)
-			return true
-		}
-		count := len(alertNames)
-		if count > 10 {
-			alertNames = append(alertNames[:10], "...")
-		}
-		if count > 0 {
-			log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
-			return true
-		}
-	}
-
-	fieldSelector := fmt.Sprintf("spec.nodeName=%s", nodeID)
-	for _, labelSelector := range podSelectors {
-		podList, err := client.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
+func (kb KubernetesBlockingChecker) isBlocked() bool {
+	fieldSelector := fmt.Sprintf("spec.nodeName=%s,status.phase!=Succeeded,status.phase!=Failed,status.phase!=Unknown", kb.nodename)
+	for _, labelSelector := range kb.filter {
+		podList, err := kb.client.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
 			LabelSelector: labelSelector,
 			FieldSelector: fieldSelector,
 			Limit:         10})
@@ -200,7 +356,15 @@ func rebootBlocked(client *kubernetes.Clientset, nodeID string) bool {
 			return true
 		}
 	}
+	return false
+}
 
+func rebootBlocked(blockers ...RebootBlocker) bool {
+	for _, blocker := range blockers {
+		if blocker.isBlocked() {
+			return true
+		}
+	}
 	return false
 }
 
@@ -230,6 +394,13 @@ func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}, TTL time.D
 	}
 }
 
+func throttle(releaseDelay time.Duration) {
+	if releaseDelay > 0 {
+		log.Infof("Delaying lock release by %v", releaseDelay)
+		time.Sleep(releaseDelay)
+	}
+}
+
 func release(lock *daemonsetlock.DaemonSetLock) {
 	log.Infof("Releasing lock")
 	if err := lock.Release(); err != nil {
@@ -242,27 +413,39 @@ func drain(client *kubernetes.Clientset, node *v1.Node) {
 
 	log.Infof("Draining node %s", nodename)
 
-	if slackHookURL != "" {
-		if err := slack.NotifyDrain(slackHookURL, slackUsername, slackChannel, messageTemplateDrain, nodename); err != nil {
-			log.Warnf("Error notifying slack: %v", err)
+	if notifyURL != "" {
+		if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateDrain, nodename)); err != nil {
+			log.Warnf("Error notifying: %v", err)
 		}
 	}
 
 	drainer := &kubectldrain.Helper{
-		Client:              client,
-		GracePeriodSeconds:  -1,
-		Force:               true,
-		DeleteLocalData:     true,
-		IgnoreAllDaemonSets: true,
-		ErrOut:              os.Stderr,
-		Out:                 os.Stdout,
+		Client:                          client,
+		Ctx:                             context.Background(),
+		GracePeriodSeconds:              drainGracePeriod,
+		SkipWaitForDeleteTimeoutSeconds: skipWaitForDeleteTimeoutSeconds,
+		Force:                           true,
+		DeleteEmptyDirData:              true,
+		IgnoreAllDaemonSets:             true,
+		ErrOut:                          os.Stderr,
+		Out:                             os.Stdout,
+		Timeout:                         drainTimeout,
 	}
+
 	if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil {
-		log.Fatal("Error cordonning %s: %v", nodename, err)
+		if !forceReboot {
+			log.Fatalf("Error cordonning %s: %v", nodename, err)
+		}
+		log.Errorf("Error cordonning %s: %v, continuing with reboot anyway", nodename, err)
+		return
 	}
 
 	if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil {
-		log.Fatal("Error draining %s: %v", nodename, err)
+		if !forceReboot {
+			log.Fatalf("Error draining %s: %v", nodename, err)
+		}
+		log.Errorf("Error draining %s: %v, continuing with reboot anyway", nodename, err)
+		return
 	}
 }
 
@@ -273,31 +456,30 @@ func uncordon(client *kubernetes.Clientset, node *v1.Node) {
 		Client: client,
 		ErrOut: os.Stderr,
 		Out:    os.Stdout,
+		Ctx:    context.Background(),
 	}
 	if err := kubectldrain.RunCordonOrUncordon(drainer, node, false); err != nil {
-		log.Fatal("Error uncordonning %s: %v", nodename, err)
+		log.Fatalf("Error uncordonning %s: %v", nodename, err)
 	}
 }
 
-func commandReboot(nodeID string) {
-	log.Infof("Commanding reboot for node: %s", nodeID)
+func invokeReboot(nodeID string, rebootCommand []string) {
+	log.Infof("Running command: %s for node: %s", rebootCommand, nodeID)
 
-	if slackHookURL != "" {
-		if err := slack.NotifyReboot(slackHookURL, slackUsername, slackChannel, messageTemplateReboot, nodeID); err != nil {
-			log.Warnf("Error notifying slack: %v", err)
+	if notifyURL != "" {
+		if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateReboot, nodeID)); err != nil {
+			log.Warnf("Error notifying: %v", err)
 		}
 	}
 
-	// Relies on hostPID:true and privileged:true to enter host mount space
-	rebootCmd := newCommand("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/bin/systemctl", "reboot")
-	if err := rebootCmd.Run(); err != nil {
+	if err := newCommand(rebootCommand[0], rebootCommand[1:]...).Run(); err != nil {
 		log.Fatalf("Error invoking reboot command: %v", err)
 	}
 }
 
-func maintainRebootRequiredMetric(nodeID string) {
+func maintainRebootRequiredMetric(nodeID string, sentinelCommand []string) {
 	for {
-		if sentinelExists() {
+		if rebootRequired(sentinelCommand) {
 			rebootRequiredGauge.WithLabelValues(nodeID).Set(1)
 		} else {
 			rebootRequiredGauge.WithLabelValues(nodeID).Set(0)
@@ -311,7 +493,45 @@ type nodeMeta struct {
 	Unschedulable bool `json:"unschedulable"`
 }
 
-func rebootAsRequired(nodeID string, window *timewindow.TimeWindow, TTL time.Duration) {
+func addNodeAnnotations(client *kubernetes.Clientset, nodeID string, annotations map[string]string) {
+	node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+	if err != nil {
+		log.Fatalf("Error retrieving node object via k8s API: %s", err)
+	}
+	for k, v := range annotations {
+		node.Annotations[k] = v
+		log.Infof("Adding node %s annotation: %s=%s", node.GetName(), k, v)
+	}
+
+	bytes, err := json.Marshal(node)
+	if err != nil {
+		log.Fatalf("Error marshalling node object into JSON: %v", err)
+	}
+
+	_, err = client.CoreV1().Nodes().Patch(context.TODO(), node.GetName(), types.StrategicMergePatchType, bytes, metav1.PatchOptions{})
+	if err != nil {
+		var annotationsErr string
+		for k, v := range annotations {
+			annotationsErr += fmt.Sprintf("%s=%s ", k, v)
+		}
+		log.Fatalf("Error adding node annotations %s via k8s API: %v", annotationsErr, err)
+	}
+}
+
+func deleteNodeAnnotation(client *kubernetes.Clientset, nodeID, key string) {
+	log.Infof("Deleting node %s annotation %s", nodeID, key)
+
+	// JSON Patch takes as path input a JSON Pointer, defined in RFC6901
+	// So we replace all instances of "/" with "~1" as per:
+	// https://tools.ietf.org/html/rfc6901#section-3
+	patch := []byte(fmt.Sprintf("[{\"op\":\"remove\",\"path\":\"/metadata/annotations/%s\"}]", strings.ReplaceAll(key, "/", "~1")))
+	_, err := client.CoreV1().Nodes().Patch(context.TODO(), nodeID, types.JSONPatchType, patch, metav1.PatchOptions{})
+	if err != nil {
+		log.Fatalf("Error deleting node annotation %s via k8s API: %v", key, err)
+	}
+}
+
+func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []string, window *timewindow.TimeWindow, TTL time.Duration, releaseDelay time.Duration) {
 	config, err := rest.InClusterConfig()
 	if err != nil {
 		log.Fatal(err)
@@ -326,44 +546,138 @@ func rebootAsRequired(nodeID string, window *timewindow.TimeWindow, TTL time.Dur
 
 	nodeMeta := nodeMeta{}
 	if holding(lock, &nodeMeta) {
+		node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+		if err != nil {
+			log.Fatalf("Error retrieving node object via k8s API: %v", err)
+		}
 		if !nodeMeta.Unschedulable {
-			node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
-			if err != nil {
-				log.Fatal(err)
-			}
 			uncordon(client, node)
 		}
+		// If we're holding the lock we know we've tried, in a prior run, to reboot
+		// So (1) we want to confirm that the reboot succeeded practically ( !rebootRequired() )
+		// And (2) check if we previously annotated the node that it was in the process of being rebooted,
+		// And finally (3) if it has that annotation, to delete it.
+		// This indicates to other node tools running on the cluster that this node may be a candidate for maintenance
+		if annotateNodes && !rebootRequired(sentinelCommand) {
+			if _, ok := node.Annotations[KuredRebootInProgressAnnotation]; ok {
+				deleteNodeAnnotation(client, nodeID, KuredRebootInProgressAnnotation)
+			}
+		}
+		throttle(releaseDelay)
 		release(lock)
 	}
 
+	preferNoScheduleTaint := taints.New(client, nodeID, preferNoScheduleTaintName, v1.TaintEffectPreferNoSchedule)
+
+	// Remove taint immediately during startup to quickly allow scheduling again.
+	if !rebootRequired(sentinelCommand) {
+		preferNoScheduleTaint.Disable()
+	}
+
+	// instantiate prometheus client
+	promClient, err := alerts.NewPromClient(papi.Config{Address: prometheusURL})
+	if err != nil {
+		log.Fatal("Unable to create prometheus client: ", err)
+	}
+
 	source := rand.NewSource(time.Now().UnixNano())
 	tick := delaytick.New(source, period)
 	for range tick {
-		if window.Contains(time.Now()) && rebootRequired() && !rebootBlocked(client, nodeID) {
-			node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
-			if err != nil {
-				log.Fatal(err)
-			}
-			nodeMeta.Unschedulable = node.Spec.Unschedulable
+		if !window.Contains(time.Now()) {
+			// Remove taint outside the reboot time window to allow for normal operation.
+			preferNoScheduleTaint.Disable()
+			continue
+		}
 
-			if acquire(lock, &nodeMeta, TTL) {
-				if !nodeMeta.Unschedulable {
-					drain(client, node)
-				}
-				commandReboot(nodeID)
-				for {
-					log.Infof("Waiting for reboot")
-					time.Sleep(time.Minute)
-				}
+		if !rebootRequired(sentinelCommand) {
+			log.Infof("Reboot not required")
+			preferNoScheduleTaint.Disable()
+			continue
+		}
+		log.Infof("Reboot required")
+
+		var blockCheckers []RebootBlocker
+		if prometheusURL != "" {
+			blockCheckers = append(blockCheckers, PrometheusBlockingChecker{promClient: promClient, filter: alertFilter, firingOnly: alertFiringOnly})
+		}
+		if podSelectors != nil {
+			blockCheckers = append(blockCheckers, KubernetesBlockingChecker{client: client, nodename: nodeID, filter: podSelectors})
+		}
+
+		if rebootBlocked(blockCheckers...) {
+			continue
+		}
+
+		node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+		if err != nil {
+			log.Fatalf("Error retrieving node object via k8s API: %v", err)
+		}
+		nodeMeta.Unschedulable = node.Spec.Unschedulable
+
+		var timeNowString string
+		if annotateNodes {
+			if _, ok := node.Annotations[KuredRebootInProgressAnnotation]; !ok {
+				timeNowString = time.Now().Format(time.RFC3339)
+				// Annotate this node to indicate that "I am going to be rebooted!"
+				// so that other node maintenance tools running on the cluster are aware that this node is in the process of a "state transition"
+				annotations := map[string]string{KuredRebootInProgressAnnotation: timeNowString}
+				// & annotate this node with a timestamp so that other node maintenance tools know how long it's been since this node has been marked for reboot
+				annotations[KuredMostRecentRebootNeededAnnotation] = timeNowString
+				addNodeAnnotations(client, nodeID, annotations)
 			}
 		}
+
+		if !acquire(lock, &nodeMeta, TTL) {
+			// Prefer to not schedule pods onto this node to avoid draing the same pod multiple times.
+			preferNoScheduleTaint.Enable()
+			continue
+		}
+
+		drain(client, node)
+
+		if rebootDelay > 0 {
+			log.Infof("Delaying reboot for %v", rebootDelay)
+			time.Sleep(rebootDelay)
+		}
+
+		invokeReboot(nodeID, rebootCommand)
+		for {
+			log.Infof("Waiting for reboot")
+			time.Sleep(time.Minute)
+		}
 	}
 }
 
+// buildSentinelCommand creates the shell command line which will need wrapping to escape
+// the container boundaries
+func buildSentinelCommand(rebootSentinelFile string, rebootSentinelCommand string) []string {
+	if rebootSentinelCommand != "" {
+		cmd, err := shlex.Split(rebootSentinelCommand)
+		if err != nil {
+			log.Fatalf("Error parsing provided sentinel command: %v", err)
+		}
+		return cmd
+	}
+	return []string{"test", "-f", rebootSentinelFile}
+}
+
+// parseRebootCommand creates the shell command line which will need wrapping to escape
+// the container boundaries
+func parseRebootCommand(rebootCommand string) []string {
+	command, err := shlex.Split(rebootCommand)
+	if err != nil {
+		log.Fatalf("Error parsing provided reboot command: %v", err)
+	}
+	return command
+}
+
 func root(cmd *cobra.Command, args []string) {
+	if logFormat == "json" {
+		log.SetFormatter(&log.JSONFormatter{})
+	}
+
 	log.Infof("Kubernetes Reboot Daemon: %s", version)
 
-	nodeID := os.Getenv("KURED_NODE_ID")
 	if nodeID == "" {
 		log.Fatal("KURED_NODE_ID environment variable required")
 	}
@@ -373,6 +687,9 @@ func root(cmd *cobra.Command, args []string) {
 		log.Fatalf("Failed to build time window: %v", err)
 	}
 
+	sentinelCommand := buildSentinelCommand(rebootSentinelFile, rebootSentinelCommand)
+	restartCommand := parseRebootCommand(rebootCommand)
+
 	log.Infof("Node ID: %s", nodeID)
 	log.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation)
 	if lockTTL > 0 {
@@ -380,12 +697,28 @@ func root(cmd *cobra.Command, args []string) {
 	} else {
 		log.Info("Lock TTL not set, lock will remain until being released")
 	}
-	log.Infof("Reboot Sentinel: %s every %v", rebootSentinel, period)
+	if lockReleaseDelay > 0 {
+		log.Infof("Lock release delay set, lock release will be delayed by: %v", lockReleaseDelay)
+	} else {
+		log.Info("Lock release delay not set, lock will be released immediately after rebooting")
+	}
+	log.Infof("PreferNoSchedule taint: %s", preferNoScheduleTaintName)
 	log.Infof("Blocking Pod Selectors: %v", podSelectors)
-	log.Infof("Reboot on: %v", window)
+	log.Infof("Reboot schedule: %v", window)
+	log.Infof("Reboot check command: %s every %v", sentinelCommand, period)
+	log.Infof("Reboot command: %s", restartCommand)
+	if annotateNodes {
+		log.Infof("Will annotate nodes during kured reboot operations")
+	}
 
-	go rebootAsRequired(nodeID, window, lockTTL)
-	go maintainRebootRequiredMetric(nodeID)
+	// To run those commands as it was the host, we'll use nsenter
+	// Relies on hostPID:true and privileged:true to enter host mount space
+	// PID set to 1, until we have a better discovery mechanism.
+	hostSentinelCommand := buildHostCommand(1, sentinelCommand)
+	hostRestartCommand := buildHostCommand(1, restartCommand)
+
+	go rebootAsRequired(nodeID, hostRestartCommand, hostSentinelCommand, window, lockTTL, lockReleaseDelay)
+	go maintainRebootRequiredMetric(nodeID, hostSentinelCommand)
 
 	http.Handle("/metrics", promhttp.Handler())
 	log.Fatal(http.ListenAndServe(":8080", nil))

cmd/kured/main_test.go

@@ -0,0 +1,235 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/weaveworks/kured/pkg/alerts"
+	assert "gotest.tools/v3/assert"
+
+	papi "github.com/prometheus/client_golang/api"
+)
+
+type BlockingChecker struct {
+	blocking bool
+}
+
+func (fbc BlockingChecker) isBlocked() bool {
+	return fbc.blocking
+}
+
+var _ RebootBlocker = BlockingChecker{}       // Verify that Type implements Interface.
+var _ RebootBlocker = (*BlockingChecker)(nil) // Verify that *Type implements Interface.
+
+func Test_flagCheck(t *testing.T) {
+	var cmd *cobra.Command
+	var args []string
+	slackHookURL = "https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"
+	flagCheck(cmd, args)
+	if notifyURL != "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET" {
+		t.Errorf("Slack URL Parsing is wrong: expecting %s  but got %s\n", "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET", notifyURL)
+	}
+}
+func Test_rebootBlocked(t *testing.T) {
+	noCheckers := []RebootBlocker{}
+	nonblockingChecker := BlockingChecker{blocking: false}
+	blockingChecker := BlockingChecker{blocking: true}
+
+	// Instantiate a prometheusClient with a broken_url
+	promClient, err := alerts.NewPromClient(papi.Config{Address: "broken_url"})
+	if err != nil {
+		log.Fatal("Can't create prometheusClient: ", err)
+	}
+	brokenPrometheusClient := PrometheusBlockingChecker{promClient: promClient, filter: nil, firingOnly: false}
+
+	type args struct {
+		blockers []RebootBlocker
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Do not block on no blocker defined",
+			args: args{blockers: noCheckers},
+			want: false,
+		},
+		{
+			name: "Ensure a blocker blocks",
+			args: args{blockers: []RebootBlocker{blockingChecker}},
+			want: true,
+		},
+		{
+			name: "Ensure a non-blocker doesn't block",
+			args: args{blockers: []RebootBlocker{nonblockingChecker}},
+			want: false,
+		},
+		{
+			name: "Ensure one blocker is enough to block",
+			args: args{blockers: []RebootBlocker{nonblockingChecker, blockingChecker}},
+			want: true,
+		},
+		{
+			name: "Do block on error contacting prometheus API",
+			args: args{blockers: []RebootBlocker{brokenPrometheusClient}},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := rebootBlocked(tt.args.blockers...); got != tt.want {
+				t.Errorf("rebootBlocked() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_buildHostCommand(t *testing.T) {
+	type args struct {
+		pid     int
+		command []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "Ensure command will run with nsenter",
+			args: args{pid: 1, command: []string{"ls", "-Fal"}},
+			want: []string{"/usr/bin/nsenter", "-m/proc/1/ns/mnt", "--", "ls", "-Fal"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := buildHostCommand(tt.args.pid, tt.args.command); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("buildHostCommand() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_buildSentinelCommand(t *testing.T) {
+	type args struct {
+		rebootSentinelFile    string
+		rebootSentinelCommand string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "Ensure a sentinelFile generates a shell 'test' command with the right file",
+			args: args{
+				rebootSentinelFile:    "/test1",
+				rebootSentinelCommand: "",
+			},
+			want: []string{"test", "-f", "/test1"},
+		},
+		{
+			name: "Ensure a sentinelCommand has priority over a sentinelFile if both are provided (because sentinelFile is always provided)",
+			args: args{
+				rebootSentinelFile:    "/test1",
+				rebootSentinelCommand: "/sbin/reboot-required -r",
+			},
+			want: []string{"/sbin/reboot-required", "-r"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := buildSentinelCommand(tt.args.rebootSentinelFile, tt.args.rebootSentinelCommand); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("buildSentinelCommand() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_parseRebootCommand(t *testing.T) {
+	type args struct {
+		rebootCommand string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "Ensure a reboot command is properly parsed",
+			args: args{
+				rebootCommand: "/sbin/systemctl reboot",
+			},
+			want: []string{"/sbin/systemctl", "reboot"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := parseRebootCommand(tt.args.rebootCommand); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseRebootCommand() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_rebootRequired(t *testing.T) {
+	type args struct {
+		sentinelCommand []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Ensure rc = 0 means reboot required",
+			args: args{
+				sentinelCommand: []string{"true"},
+			},
+			want: true,
+		},
+		{
+			name: "Ensure rc != 0 means reboot NOT required",
+			args: args{
+				sentinelCommand: []string{"false"},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := rebootRequired(tt.args.sentinelCommand); got != tt.want {
+				t.Errorf("rebootRequired() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_rebootRequired_fatals(t *testing.T) {
+	cases := []struct {
+		param       []string
+		expectFatal bool
+	}{
+		{
+			param:       []string{"true"},
+			expectFatal: false,
+		},
+		{
+			param:       []string{"./babar"},
+			expectFatal: true,
+		},
+	}
+
+	defer func() { log.StandardLogger().ExitFunc = nil }()
+	var fatal bool
+	log.StandardLogger().ExitFunc = func(int) { fatal = true }
+
+	for _, c := range cases {
+		fatal = false
+		rebootRequired(c.param)
+		assert.Equal(t, c.expectFatal, fatal)
+	}
+
+}

go.mod

@@ -1,14 +1,20 @@
 module github.com/weaveworks/kured
 
-go 1.15
+go 1.16
 
 require (
-	github.com/prometheus/client_golang v1.8.0
-	github.com/prometheus/common v0.15.0
-	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/cobra v1.1.1
-	k8s.io/api v0.19.4
-	k8s.io/apimachinery v0.19.4
-	k8s.io/client-go v0.19.4
-	k8s.io/kubectl v0.19.4
+	github.com/containrrr/shoutrrr v0.5.2
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/common v0.32.1
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cobra v1.3.0
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.10.1
+	github.com/stretchr/testify v1.7.0
+	gotest.tools/v3 v3.0.3
+	k8s.io/api v0.22.4
+	k8s.io/apimachinery v0.22.4
+	k8s.io/client-go v0.22.4
+	k8s.io/kubectl v0.22.4
 )

kured-ds.yaml

@@ -29,7 +29,7 @@ spec:
       restartPolicy: Always
       containers:
         - name: kured
-          image: docker.io/weaveworks/kured
+          image: docker.io/weaveworks/kured:1.9.1
                  # If you find yourself here wondering why there is no
                  # :latest tag on Docker Hub,see the FAQ in the README
           imagePullPolicy: IfNotPresent
@@ -44,22 +44,35 @@ spec:
                   fieldPath: spec.nodeName
           command:
             - /usr/bin/kured
-#            - --alert-filter-regexp=^RebootRequired$
-#            - --blocking-pod-selector=runtime=long,cost=expensive
-#            - --blocking-pod-selector=name=temperamental
-#            - --blocking-pod-selector=...
-#            - --ds-name=kured
-#            - --ds-namespace=kube-system
-#            - --end-time=23:59:59
-#            - --lock-annotation=weave.works/kured-node-lock
+#            - --force-reboot=false
+#            - --drain-grace-period=-1
+#            - --skip-wait-for-delete-timeout=0
+#            - --drain-timeout=0
 #            - --period=1h
+#            - --ds-namespace=kube-system
+#            - --ds-name=kured
+#            - --lock-annotation=weave.works/kured-node-lock
+#            - --lock-ttl=0
 #            - --prometheus-url=http://prometheus.monitoring.svc.cluster.local
-#            - --reboot-days=sun,mon,tue,wed,thu,fri,sat
+#            - --alert-filter-regexp=^RebootRequired$
+#            - --alert-firing-only=false
 #            - --reboot-sentinel=/var/run/reboot-required
+#            - --prefer-no-schedule-taint=""
+#            - --reboot-sentinel-command=""
 #            - --slack-hook-url=https://hooks.slack.com/...
 #            - --slack-username=prod
 #            - --slack-channel=alerting
+#            - --notify-url="" # See also shoutrrr url format
 #            - --message-template-drain=Draining node %s
 #            - --message-template-drain=Rebooting node %s
+#            - --blocking-pod-selector=runtime=long,cost=expensive
+#            - --blocking-pod-selector=name=temperamental
+#            - --blocking-pod-selector=...
+#            - --reboot-days=sun,mon,tue,wed,thu,fri,sat
+#            - --reboot-delay=90s
 #            - --start-time=0:00
+#            - --end-time=23:59:59
 #            - --time-zone=UTC
+#            - --annotate-nodes=false
+#            - --lock-release-delay=30m
+#            - --log-format=text

pkg/alerts/prometheus.go

@@ -7,22 +7,39 @@ import (
 	"sort"
 	"time"
 
-	"github.com/prometheus/client_golang/api"
+	papi "github.com/prometheus/client_golang/api"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/common/model"
 )
 
-// PrometheusActiveAlerts returns a list of names of active (e.g. pending or firing) alerts, filtered
-// by the supplied regexp.
-func PrometheusActiveAlerts(prometheusURL string, filter *regexp.Regexp) ([]string, error) {
-	client, err := api.NewClient(api.Config{Address: prometheusURL})
+// PromClient is a wrapper around the Prometheus Client interface and implements the api
+// This way, the PromClient can be instantiated with the configuration the Client needs, and
+// the ability to use the methods the api has, like Query and so on.
+type PromClient struct {
+	papi papi.Client
+	api  v1.API
+}
+
+// NewPromClient creates a new client to the Prometheus API.
+// It returns an error on any problem.
+func NewPromClient(conf papi.Config) (*PromClient, error) {
+	promClient, err := papi.NewClient(conf)
 	if err != nil {
 		return nil, err
 	}
+	client := PromClient{papi: promClient, api: v1.NewAPI(promClient)}
+	return &client, nil
+}
 
-	queryAPI := v1.NewAPI(client)
+// ActiveAlerts is a method of type PromClient, it returns a list of names of active alerts
+// (e.g. pending or firing), filtered by the supplied regexp or by the includeLabels query.
+// filter by regexp means when the regex finds the alert-name; the alert is exluded from the
+// block-list and will NOT block rebooting. query by includeLabel means,
+// if the query finds an alert, it will include it to the block-list and it WILL block rebooting.
+func (p *PromClient) ActiveAlerts(filter *regexp.Regexp, firingOnly bool) ([]string, error) {
 
-	value, _, err := queryAPI.Query(context.Background(), "ALERTS", time.Now())
+	// get all alerts from prometheus
+	value, _, err := p.api.Query(context.Background(), "ALERTS", time.Now())
 	if err != nil {
 		return nil, err
 	}
@@ -32,7 +49,7 @@ func PrometheusActiveAlerts(prometheusURL string, filter *regexp.Regexp) ([]stri
 			activeAlertSet := make(map[string]bool)
 			for _, sample := range vector {
 				if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 {
-					if filter == nil || !filter.MatchString(string(alertName)) {
+					if (filter == nil || !filter.MatchString(string(alertName))) && (!firingOnly || sample.Metric["alertstate"] == "firing") {
 						activeAlertSet[string(alertName)] = true
 					}
 				}
@@ -42,7 +59,7 @@ func PrometheusActiveAlerts(prometheusURL string, filter *regexp.Regexp) ([]stri
 			for activeAlert := range activeAlertSet {
 				activeAlerts = append(activeAlerts, activeAlert)
 			}
-			sort.Sort(sort.StringSlice(activeAlerts))
+			sort.Strings(activeAlerts)
 
 			return activeAlerts, nil
 		}

pkg/alerts/prometheus_test.go

@@ -0,0 +1,141 @@
+package alerts
+
+import (
+	"log"
+	"net/http"
+	"net/http/httptest"
+
+	"regexp"
+	"testing"
+
+	"github.com/prometheus/client_golang/api"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type MockResponse struct {
+	StatusCode int
+	Body       []byte
+}
+
+// MockServerProperties ties a mock response to a url and a method
+type MockServerProperties struct {
+	URI        string
+	HTTPMethod string
+	Response   MockResponse
+}
+
+// NewMockServer sets up a new MockServer with properties ad starts the server.
+func NewMockServer(props ...MockServerProperties) *httptest.Server {
+
+	handler := http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			for _, proc := range props {
+				_, err := w.Write(proc.Response.Body)
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+		})
+	return httptest.NewServer(handler)
+}
+
+func TestActiveAlerts(t *testing.T) {
+	responsebody := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"GatekeeperViolations","alertstate":"firing","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"PodCrashing-dev","alertstate":"firing","container":"deployment","instance":"1.2.3.4:8080","job":"kube-state-metrics","namespace":"dev","pod":"dev-deployment-78dcbmf25v","severity":"critical","team":"dev"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"PodRestart-dev","alertstate":"firing","container":"deployment","instance":"1.2.3.4:1234","job":"kube-state-metrics","namespace":"qa","pod":"qa-job-deployment-78dcbmf25v","severity":"warning","team":"qa"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"PrometheusTargetDown","alertstate":"firing","job":"kubernetes-pods","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"ScheduledRebootFailing","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`
+	addr := "http://localhost:10001"
+
+	for _, tc := range []struct {
+		it         string
+		rFilter    string
+		respBody   string
+		aName      string
+		wantN      int
+		firingOnly bool
+	}{
+		{
+			it:         "should return no active alerts",
+			respBody:   responsebody,
+			rFilter:    "",
+			wantN:      0,
+			firingOnly: false,
+		},
+		{
+			it:         "should return a subset of all alerts",
+			respBody:   responsebody,
+			rFilter:    "Pod",
+			wantN:      3,
+			firingOnly: false,
+		},
+		{
+			it:         "should return all active alerts by regex",
+			respBody:   responsebody,
+			rFilter:    "*",
+			wantN:      5,
+			firingOnly: false,
+		},
+		{
+			it:         "should return all active alerts by regex filter",
+			respBody:   responsebody,
+			rFilter:    "*",
+			wantN:      5,
+			firingOnly: false,
+		},
+		{
+			it:         "should return only firing alerts if firingOnly is true",
+			respBody:   responsebody,
+			rFilter:    "*",
+			wantN:      4,
+			firingOnly: true,
+		},
+		{
+			it:         "should return ScheduledRebootFailing active alerts",
+			respBody:   `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"ScheduledRebootFailing","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`,
+			aName:      "ScheduledRebootFailing",
+			rFilter:    "*",
+			wantN:      1,
+			firingOnly: false,
+		},
+		{
+			it:         "should not return an active alert if RebootRequired is firing (regex filter)",
+			respBody:   `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"RebootRequired","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`,
+			rFilter:    "RebootRequired",
+			wantN:      0,
+			firingOnly: false,
+		},
+	} {
+		// Start mockServer
+		mockServer := NewMockServer(MockServerProperties{
+			URI:        addr,
+			HTTPMethod: http.MethodPost,
+			Response: MockResponse{
+				Body: []byte(tc.respBody),
+			},
+		})
+		// Close mockServer after all connections are gone
+		defer mockServer.Close()
+
+		t.Run(tc.it, func(t *testing.T) {
+
+			// regex filter
+			regex, _ := regexp.Compile(tc.rFilter)
+
+			// instantiate the prometheus client with the mockserver-address
+			p, err := NewPromClient(api.Config{Address: mockServer.URL})
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			result, err := p.ActiveAlerts(regex, tc.firingOnly)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			// assert
+			assert.Equal(t, tc.wantN, len(result), "expected amount of alerts %v, got %v", tc.wantN, len(result))
+
+			if tc.aName != "" {
+				assert.Equal(t, tc.aName, result[0], "expected active alert %v, got %v", tc.aName, result[0])
+			}
+		})
+	}
+}

pkg/daemonsetlock/daemonsetlock.go

@@ -6,11 +6,18 @@ import (
 	"fmt"
 	"time"
 
+	v1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 )
 
+const (
+	k8sAPICallRetrySleep   = 5 * time.Second // How much time to wait in between retrying a k8s API call
+	k8sAPICallRetryTimeout = 5 * time.Minute // How long to wait until we determine that the k8s API is definitively unavailable
+)
+
 // DaemonSetLock holds all necessary information to do actions
 // on the kured ds which holds lock info through annotations.
 type DaemonSetLock struct {
@@ -34,11 +41,11 @@ func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation strin
 }
 
 // Acquire attempts to annotate the kured daemonset with lock info from instantiated DaemonSetLock using client-go
-func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acquired bool, owner string, err error) {
+func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (bool, string, error) {
 	for {
-		ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{})
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
-			return false, "", err
+			return false, "", fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 		}
 
 		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
@@ -78,10 +85,10 @@ func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acqu
 }
 
 // Test attempts to check the kured daemonset lock status (existence, expiry) from instantiated DaemonSetLock using client-go
-func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) {
-	ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{})
+func (dsl *DaemonSetLock) Test(metadata interface{}) (bool, error) {
+	ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 	if err != nil {
-		return false, err
+		return false, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 	}
 
 	valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
@@ -102,9 +109,9 @@ func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) {
 // Release attempts to remove the lock data from the kured ds annotations using client-go
 func (dsl *DaemonSetLock) Release() error {
 	for {
-		ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{})
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
-			return err
+			return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 		}
 
 		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
@@ -137,6 +144,24 @@ func (dsl *DaemonSetLock) Release() error {
 	}
 }
 
+// GetDaemonSet returns the named DaemonSet resource from the DaemonSetLock's configured client
+func (dsl *DaemonSetLock) GetDaemonSet(sleep, timeout time.Duration) (*v1.DaemonSet, error) {
+	var ds *v1.DaemonSet
+	var lastError error
+	err := wait.PollImmediate(sleep, timeout, func() (bool, error) {
+		ctx, cancel := context.WithTimeout(context.Background(), timeout)
+		defer cancel()
+		if ds, lastError = dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(ctx, dsl.name, metav1.GetOptions{}); lastError != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Timed out trying to get daemonset %s in namespace %s: %v", dsl.name, dsl.namespace, lastError)
+	}
+	return ds, nil
+}
+
 func ttlExpired(created time.Time, ttl time.Duration) bool {
 	if ttl > 0 && time.Since(created) >= ttl {
 		return true

pkg/notifications/slack/slack.go

@@ -1,54 +0,0 @@
-package slack
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"time"
-)
-
-var (
-	httpClient = &http.Client{Timeout: 5 * time.Second}
-)
-
-type body struct {
-	Text     string `json:"text,omitempty"`
-	Username string `json:"username,omitempty"`
-	Channel  string `json:"channel,omitempty"`
-}
-
-func notify(hookURL, username, channel, message string) error {
-	msg := body{
-		Text:     message,
-		Username: username,
-		Channel:  channel,
-	}
-
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(&msg); err != nil {
-		return err
-	}
-
-	resp, err := httpClient.Post(hookURL, "application/json", &buf)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf(resp.Status)
-	}
-
-	return nil
-}
-
-// NotifyDrain is the exposed way to notify of a drain event onto a slack chan
-func NotifyDrain(hookURL, username, channel, messageTemplate, nodeID string) error {
-	return notify(hookURL, username, channel, fmt.Sprintf(messageTemplate, nodeID))
-}
-
-// NotifyReboot is the exposed way to notify of a reboot event onto a slack chan
-func NotifyReboot(hookURL, username, channel, messageTemplate, nodeID string) error {
-	return notify(hookURL, username, channel, fmt.Sprintf(messageTemplate, nodeID))
-}

pkg/taints/taints.go

@@ -0,0 +1,166 @@
+package taints
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Taint allows to set soft and hard limitations for scheduling and executing pods on nodes.
+type Taint struct {
+	client    *kubernetes.Clientset
+	nodeID    string
+	taintName string
+	effect    v1.TaintEffect
+	exists    bool
+}
+
+// New provides a new taint.
+func New(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect) *Taint {
+	exists, _, _ := taintExists(client, nodeID, taintName)
+
+	return &Taint{
+		client:    client,
+		nodeID:    nodeID,
+		taintName: taintName,
+		effect:    effect,
+		exists:    exists,
+	}
+}
+
+// Enable creates the taint for a node. Creating an existing taint is a noop.
+func (t *Taint) Enable() {
+	if t.taintName == "" {
+		return
+	}
+
+	if t.exists {
+		return
+	}
+
+	preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, true)
+
+	t.exists = true
+}
+
+// Disable removes the taint for a node. Removing a missing taint is a noop.
+func (t *Taint) Disable() {
+	if t.taintName == "" {
+		return
+	}
+
+	if !t.exists {
+		return
+	}
+
+	preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, false)
+
+	t.exists = false
+}
+
+func taintExists(client *kubernetes.Clientset, nodeID, taintName string) (bool, int, *v1.Node) {
+	updatedNode, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+	if err != nil || updatedNode == nil {
+		log.Fatalf("Error reading node %s: %v", nodeID, err)
+	}
+
+	for i, taint := range updatedNode.Spec.Taints {
+		if taint.Key == taintName {
+			return true, i, updatedNode
+		}
+	}
+
+	return false, 0, updatedNode
+}
+
+func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect, shouldExists bool) {
+	taintExists, offset, updatedNode := taintExists(client, nodeID, taintName)
+
+	if taintExists && shouldExists {
+		log.Debugf("Taint %v exists already for node %v.", taintName, nodeID)
+		return
+	}
+
+	if !taintExists && !shouldExists {
+		log.Debugf("Taint %v already missing for node %v.", taintName, nodeID)
+		return
+	}
+
+	type patchTaints struct {
+		Op    string      `json:"op"`
+		Path  string      `json:"path"`
+		Value interface{} `json:"value,omitempty"`
+	}
+
+	taint := v1.Taint{
+		Key:    taintName,
+		Effect: effect,
+	}
+
+	var patches []patchTaints
+
+	if len(updatedNode.Spec.Taints) == 0 {
+		// add first taint and ensure to keep current taints
+		patches = []patchTaints{
+			{
+				Op:    "test",
+				Path:  "/spec",
+				Value: updatedNode.Spec,
+			},
+			{
+				Op:    "add",
+				Path:  "/spec/taints",
+				Value: []v1.Taint{},
+			},
+			{
+				Op:    "add",
+				Path:  "/spec/taints/-",
+				Value: taint,
+			},
+		}
+	} else if taintExists {
+		// remove taint and ensure to test against race conditions
+		patches = []patchTaints{
+			{
+				Op:    "test",
+				Path:  fmt.Sprintf("/spec/taints/%d", offset),
+				Value: taint,
+			},
+			{
+				Op:   "remove",
+				Path: fmt.Sprintf("/spec/taints/%d", offset),
+			},
+		}
+	} else {
+		// add missing taint to exsting list
+		patches = []patchTaints{
+			{
+				Op:    "add",
+				Path:  "/spec/taints/-",
+				Value: taint,
+			},
+		}
+	}
+
+	patchBytes, err := json.Marshal(patches)
+	if err != nil {
+		log.Fatalf("Error encoding taint patch for node %s: %v", nodeID, err)
+	}
+
+	_, err = client.CoreV1().Nodes().Patch(context.TODO(), nodeID, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
+	if err != nil {
+		log.Fatalf("Error patching taint for node %s: %v", nodeID, err)
+	}
+
+	if shouldExists {
+		log.Info("Node taint added")
+	} else {
+		log.Info("Node taint removed")
+	}
+}

tests/kind/test-metrics.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+expected="$1"
+if [[ "$expected" != "0" && "$expected" != "1" ]]; then
+    echo "You should give an argument to this script, the gauge value (0 or 1)"
+    exit 1
+fi
+
+HOST="${HOST:-localhost}"
+PORT="${PORT:-30000}"
+NODENAME="${NODENAME-chart-testing-control-plane}"
+
+reboot_required=$(docker exec "$NODENAME" curl "http://$HOST:$PORT/metrics" | awk '/^kured_reboot_required/{print $2}')
+if [[ "$reboot_required" == "$expected" ]]; then
+    echo "Test success"
+else
+    echo "Test failed"
+    exit 1
+fi