feat: update to 1.15.0 (#887)

Signed-off-by: Christian Kotzbauer <git@ckotzbauer.de>

.clomonitor.yml

@@ -0,0 +1,3 @@
+exemptions:
+  - check: analytics
+    reason: "We don't track people"

.github/ct.yaml

@@ -1,6 +0,0 @@
-# See https://github.com/helm/chart-testing#configuration
-remote: origin
-chart-dirs:
-  - charts
-chart-repos: []
-helm-extra-args: --timeout 600s

.github/dependabot.yml

@@ -15,3 +15,7 @@ updates:
       - dependency-name: "k8s.io/apimachinery"
       - dependency-name: "k8s.io/client-go"
       - dependency-name: "k8s.io/kubectl"
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/kind-cluster-1.18.yaml

@@ -1,13 +0,0 @@
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  image: kindest/node:v1.18.8
-- role: control-plane
-  image: kindest/node:v1.18.8
-- role: control-plane
-  image: kindest/node:v1.18.8
-- role: worker
-  image: kindest/node:v1.18.8
-- role: worker
-  image: kindest/node:v1.18.8

.github/kind-cluster-1.19.yaml

@@ -1,13 +0,0 @@
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  image: kindest/node:v1.19.4
-- role: control-plane
-  image: kindest/node:v1.19.4
-- role: control-plane
-  image: kindest/node:v1.19.4
-- role: worker
-  image: kindest/node:v1.19.4
-- role: worker
-  image: kindest/node:v1.19.4

.github/kind-cluster-1.20.yaml

@@ -1,13 +0,0 @@
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  image: "kindest/node:v1.20.0"
-- role: control-plane
-  image: "kindest/node:v1.20.0"
-- role: control-plane
-  image: "kindest/node:v1.20.0"
-- role: worker
-  image: "kindest/node:v1.20.0"
-- role: worker
-  image: "kindest/node:v1.20.0"

.github/kind-cluster-1.27.yaml

@@ -0,0 +1,13 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: "kindest/node:v1.27.3"
+- role: control-plane
+  image: "kindest/node:v1.27.3"
+- role: control-plane
+  image: "kindest/node:v1.27.3"
+- role: worker
+  image: "kindest/node:v1.27.3"
+- role: worker
+  image: "kindest/node:v1.27.3"

.github/kind-cluster-1.28.yaml

@@ -0,0 +1,13 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: "kindest/node:v1.28.0"
+- role: control-plane
+  image: "kindest/node:v1.28.0"
+- role: control-plane
+  image: "kindest/node:v1.28.0"
+- role: worker
+  image: "kindest/node:v1.28.0"
+- role: worker
+  image: "kindest/node:v1.28.0"

.github/kind-cluster-1.29.yaml

@@ -0,0 +1,13 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: "kindest/node:v1.29.0"
+- role: control-plane
+  image: "kindest/node:v1.29.0"
+- role: control-plane
+  image: "kindest/node:v1.29.0"
+- role: worker
+  image: "kindest/node:v1.29.0"
+- role: worker
+  image: "kindest/node:v1.29.0"

.github/scripts/goreleaser-install.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+set -e
+
+RELEASES_URL="https://github.com/goreleaser/goreleaser/releases"
+FILE_BASENAME="goreleaser"
+
+test -z "$VERSION" && {
+    echo "Unable to get goreleaser version." >&2
+    exit 1
+}
+
+test -z "$TMPDIR" && TMPDIR="$(mktemp -d)"
+# goreleaser uses arm64 instead of aarch64
+goreleaser_arch=$(uname -m | sed -e 's/aarch64/arm64/g' -e 's/ppc64le/ppc64/' -e 's/armv7l/armv7/' )
+TAR_FILE="$TMPDIR/${FILE_BASENAME}_$(uname -s)_${goreleaser_arch}.tar.gz"
+export TAR_FILE
+
+(
+    echo "Downloading GoReleaser $VERSION..."
+    curl -sfLo "$TAR_FILE" \
+        "$RELEASES_URL/download/$VERSION/${FILE_BASENAME}_$(uname -s)_${goreleaser_arch}.tar.gz"
+    cd "$TMPDIR"
+    curl -sfLo "checksums.txt" "$RELEASES_URL/download/$VERSION/checksums.txt"
+    echo "Verifying checksums..."
+    sha256sum --ignore-missing --quiet --check checksums.txt
+)
+
+tar -xf "$TAR_FILE" -O goreleaser > "$TMPDIR/goreleaser"
+rm "$TMPDIR/checksums.txt"
+rm "$TAR_FILE"

.github/workflows/codeql.yml

@@ -0,0 +1,75 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "main" ]
+  schedule:
+    - cron: '24 13 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/on-main-push.yaml

@@ -0,0 +1,82 @@
+# We publish every merged commit in the form of an image
+# named kured:<branch>-<short tag>
+name: Push image of latest main
+on:
+  push:
+    branches:
+      - main
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  tag-scan-and-push-final-image:
+    name: "Build, scan, and publish tagged image"
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Ensure go version
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
+
+      - name: Setup GoReleaser
+        run: make bootstrap-tools
+
+      - name: Build binaries
+        run: make kured-release-snapshot
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/arm64, linux/amd64, linux/arm/v7, linux/arm/v6, linux/386
+          push: true
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
+
+      - name: Generate SBOM
+        run: |
+          .tmp/syft ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }} -o spdx > kured.sbom
+
+      - name: Sign and attest artifacts
+        run: |
+          .tmp/cosign sign -y -r ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
+
+          .tmp/cosign sign-blob -y --output-signature kured.sbom.sig --output-certificate kured.sbom.pem kured.sbom
+
+          .tmp/cosign attest -y --type spdx --predicate kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
+          .tmp/cosign attach sbom --type spdx --sbom kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}

.github/workflows/on-master-push-charts.yaml

@@ -1,19 +0,0 @@
-name: Publish helm chart
-on:
-  push:
-    branches:
-      - "master"
-    paths:
-      - "charts/**"
-
-jobs:
-  publish-helm-chart:
-    name: Publish latest chart
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Publish Helm chart
-        uses: stefanprodan/helm-gh-pages@master
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          charts_dir: charts

.github/workflows/on-master-push.yaml

@@ -1,38 +0,0 @@
-# We publish every merged commit in the form of an image
-# named kured:<branch>-<short tag>
-name: Push image of latest master
-on:
-  push:
-    branches:
-      - master
-jobs:
-  tag-scan-and-push-final-image:
-    name: "Build, scan, and publish tagged image"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-
-      - name: Find go version
-        run: |
-          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-          echo "::set-output name=version::${GO_VERSION}"
-        id: awk_gomod
-
-      - name: Ensure go version
-        uses: actions/setup-go@v2
-        with:
-          go-version: "${{ steps.awk_gomod.outputs.version }}"
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME_WEAVEWORKSKUREDCI }}
-          password: ${{ secrets.DOCKERHUB_TOKEN_WEAVEWORKSKUREDCI }}
-
-      - name: Build image
-        run: |
-          make DH_ORG="${{ github.repository_owner }}" image
-
-      - name: Publish image
-        run: |
-          make DH_ORG="${{ github.repository_owner }}" publish-image

.github/workflows/on-pr-charts.yaml

@@ -1,41 +0,0 @@
-#This is just extra testing, for lint check, and basic installation
-#If those fail, no need to test the rest of the PR (github will cancel the rest of the builds)
-name: PR - charts
-on:
-  pull_request:
-    paths:
-      - "charts/**"
-jobs:
-  # We create two jobs (with a matrix) instead of one to make those parallel.
-  # We don't need to conditionally check if something has changed, due to github actions
-  # tackling that for us.
-  # Fail-fast ensures that if one of those matrix job fail, the other one gets cancelled.
-  test-chart:
-    name: Test helm chart
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: true
-      matrix:
-        test-action:
-          - lint
-          - install
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-
-      # Helm is already present in github actions, so do not re-install it
-      - name: Setup chart testing
-        uses: helm/chart-testing-action@v2.0.1
-
-      - name: Create default kind cluster
-        uses: helm/kind-action@v1.1.0
-        if: ${{ matrix.test-action == 'install' }}
-
-      - name: Run chart tests 
-        run: ct ${{ matrix.test-action }} --config .github/ct.yaml

.github/workflows/on-pr.yaml

@@ -4,11 +4,30 @@ on:
   push:
 
 jobs:
+  pr-gotest:
+    name: Run go tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Ensure go version
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: run tests
+        run: go test -json ./... > test.json
+      - name: Annotate tests
+        if: always()
+        uses: guyarb/golang-test-annoations@v0.7.0
+        with:
+          test-results: test.json
+
   pr-shellcheck:
     name: Lint bash code with shellcheck
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run ShellCheck
       uses: bewuethr/shellcheck-action@v2
 
@@ -16,22 +35,18 @@ jobs:
     name: Lint golang code
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Find go version
-      run: |
-        GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-        echo "::set-output name=version::${GO_VERSION}"
-      id: awk_gomod
+    - uses: actions/checkout@v4
     - name: Ensure go version
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
-        go-version: "${{ steps.awk_gomod.outputs.version }}"
+        go-version-file: 'go.mod'
+        check-latest: true
     - name: Lint cmd folder
-      uses: Jerome1337/golint-action@v1.0.2
+      uses: Jerome1337/golint-action@v1.0.3
       with:
         golint-path: './cmd/...'
     - name: Lint pkg folder
-      uses: Jerome1337/golint-action@v1.0.2
+      uses: Jerome1337/golint-action@v1.0.3
       with:
         golint-path: './pkg/...'
 
@@ -39,14 +54,14 @@ jobs:
     name: Check docs for incorrect links
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Link Checker
-      id: lc
-      uses: peter-evans/link-checker@v1
+      uses: lycheeverse/lychee-action@c3089c702fbb949e3f7a8122be0c33c017904f9b
+      env:
+        GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
       with:
-        args: -r *.md *.yaml */*/*.go -x .cluster.local
-    - name: Fail if there were link errors
-      run: exit ${{ steps.lc.outputs.exit_code }}
+        args: --verbose --no-progress '*.md' '*.yaml' '*/*/*.go' --exclude-link-local
+        fail: true
 
   # This should not be made a mandatory test
   # It is only used to make us aware of any potential security failure, that
@@ -55,48 +70,72 @@ jobs:
     name: Build image and scan it against known vulnerabilities
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Find go version
-        run: |
-          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-          echo "::set-output name=version::${GO_VERSION}"
-        id: awk_gomod
+      - uses: actions/checkout@v4
       - name: Ensure go version
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: "${{ steps.awk_gomod.outputs.version }}"
-      - run: make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
-      - uses: Azure/container-scan@v0
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Setup GoReleaser
+        run: make bootstrap-tools
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
+      - name: Build image
+        run: VERSION="${{ steps.tags.outputs.sha_short }}" make image
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@d43c1f16c00cfd3978dde6c07f4bbcf9eb6993ca
         with:
-          image-name: docker.io/${{ github.repository_owner }}/kured:${{ github.sha }}
+          image-ref: 'ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }}'
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
 
-  # If the PRs don't break the behaviour in the helm chart, we can simply publish the helm chart at the time of the release.
-  e2e-helm:
-    name: "Functional test of helm chart, e2e testing"
+  # This ensures the latest code works with the manifests built from tree.
+  # It is useful for two things:
+  # - Test manifests changes (obviously), ensuring they don't break existing clusters
+  # - Ensure manifests work with the latest versions even with no manifest change
+  #     (compared to helm charts, manifests cannot easily template changes based on versions)
+  # Helm charts are _trailing_ releases, while manifests are done during development.
+  # This test uses the "command" reboot-method.
+  e2e-manifests-command:
+    name: End-to-End test with kured with code and manifests from HEAD (command)
     runs-on: ubuntu-latest
-    # only build with oldest and newest supported, it should be good enough.
     strategy:
+      fail-fast: false
       matrix:
         kubernetes:
-          - "1.18"
-          - "1.20"
+          - "1.27"
+          - "1.28"
+          - "1.29"
     steps:
-      - uses: actions/checkout@v2
-      - name: Find go version
-        run: |
-          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-          echo "::set-output name=version::${GO_VERSION}"
-        id: awk_gomod
+      - uses: actions/checkout@v4
       - name: Ensure go version
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: "${{ steps.awk_gomod.outputs.version }}"
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Setup GoReleaser
+        run: make bootstrap-tools
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
       - name: Build artifacts
         run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart
+          VERSION="${{ steps.tags.outputs.sha_short }}" make image
+          VERSION="${{ steps.tags.outputs.sha_short }}" make manifest
 
-      - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions"
+      - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions
         run: |
           sudo bash << EOF
               cp /etc/docker/daemon.json /etc/docker/daemon.json.old
@@ -106,33 +145,31 @@ jobs:
           EOF
 
       # Default name for helm/kind-action kind clusters is "chart-testing"
-      - name: Create 5 node kind cluster
-        uses: helm/kind-action@master
+      - name: Create kind cluster with 5 nodes
+        uses: helm/kind-action@v1.8.0
         with:
           config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
+          version: v0.14.0
 
       - name: Preload previously built images onto kind cluster
-        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
+        run: kind load docker-image ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }} --name chart-testing
 
-      - name: Deploy kured on default namespace with its helm chart
+      - name: Do not wait for an hour before detecting the rebootSentinel
         run: |
-          # Documented in official helm doc to live on the edge
-          curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
-          # Refresh bins
-          hash -r
-          helm install kured ./charts/kured/ --set configuration.period=1m
-          kubectl config set-context kind-chart-testing
-          kubectl get ds --all-namespaces
-          kubectl describe ds kured
+          sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds.yaml
+
+      - name: Install kured with kubectl
+        run: |
+          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml
 
       - name: Ensure kured is ready
-        uses: nick-invision/retry@v2.4.0
+        uses: nick-invision/retry@v2.9.0
         with:
           timeout_minutes: 10
           max_attempts: 10
           retry_wait_seconds: 60
-          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = 5
-          command: "kubectl get ds kured | grep -E 'kured.*5.*5.*5.*5.*5' "
+          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
+          command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
 
       - name: Create reboot sentinel files
         run: |
@@ -144,31 +181,45 @@ jobs:
         run: |
           ./tests/kind/follow-coordinated-reboot.sh
 
-  # This workflow is useful when introducing new versions, to ensure our manifests
-  # still work (even if there might be no manifest 'code' change).
-  # The version used here is what hasn't been tested with the helm chart
-  deploy-manifests:
-    name: Deploy kured with current manifests
+
+  # This ensures the latest code works with the manifests built from tree.
+  # It is useful for two things:
+  # - Test manifests changes (obviously), ensuring they don't break existing clusters
+  # - Ensure manifests work with the latest versions even with no manifest change
+  #     (compared to helm charts, manifests cannot easily template changes based on versions)
+  # Helm charts are _trailing_ releases, while manifests are done during development.
+  # This test uses the "signal" reboot-method.
+  e2e-manifests-signal:
+    name: End-to-End test with kured with code and manifests from HEAD (signal)
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         kubernetes:
-          - 1.18
+          - "1.27"
+          - "1.28"
+          - "1.29"
     steps:
-      - uses: actions/checkout@v2
-      - name: Find go version
-        run: |
-          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-          echo "::set-output name=version::${GO_VERSION}"
-        id: awk_gomod
+      - uses: actions/checkout@v4
       - name: Ensure go version
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: "${{ steps.awk_gomod.outputs.version }}"
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Setup GoReleaser
+        run: make bootstrap-tools
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
       - name: Build artifacts
         run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" manifest
+          VERSION="${{ steps.tags.outputs.sha_short }}" make image
+          VERSION="${{ steps.tags.outputs.sha_short }}" make manifest
+
       - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions
         run: |
           sudo bash << EOF
@@ -177,21 +228,127 @@ jobs:
               systemctl restart docker || journalctl --no-pager -n 500
               systemctl status docker
           EOF
+
       # Default name for helm/kind-action kind clusters is "chart-testing"
-      - name: Create kind cluster
-        uses: helm/kind-action@master
+      - name: Create kind cluster with 5 nodes
+        uses: helm/kind-action@v1.8.0
         with:
           config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
+          version: v0.14.0
+
       - name: Preload previously built images onto kind cluster
-        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
+        run: kind load docker-image ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }} --name chart-testing
+
+      - name: Do not wait for an hour before detecting the rebootSentinel
+        run: |
+          sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds-signal.yaml
+
       - name: Install kured with kubectl
         run: |
-          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml
+          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds-signal.yaml
+
       - name: Ensure kured is ready
-        uses: nick-invision/retry@v2.4.0
+        uses: nick-invision/retry@v2.9.0
         with:
           timeout_minutes: 10
           max_attempts: 10
           retry_wait_seconds: 60
           # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
           command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
+
+      - name: Create reboot sentinel files
+        run: |
+          ./tests/kind/create-reboot-sentinels.sh
+
+      - name: Follow reboot until success
+        env:
+          DEBUG: true
+        run: |
+          ./tests/kind/follow-coordinated-reboot.sh
+
+
+
+  # This ensures the latest code works with the manifests built from tree.
+  # It is useful for two things:
+  # - Test manifests changes (obviously), ensuring they don't break existing clusters
+  # - Ensure manifests work with the latest versions even with no manifest change
+  #     (compared to helm charts, manifests cannot easily template changes based on versions)
+  # Helm charts are _trailing_ releases, while manifests are done during development.
+  # Concurrency = 2
+  e2e-manifests-concurent:
+    name: End-to-End test with kured with code and manifests from HEAD (concurrent)
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        kubernetes:
+          - "1.27"
+          - "1.28"
+          - "1.29"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Ensure go version
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Setup GoReleaser
+        run: make bootstrap-tools
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
+      - name: Build artifacts
+        run: |
+          VERSION="${{ steps.tags.outputs.sha_short }}" make image
+          VERSION="${{ steps.tags.outputs.sha_short }}" make manifest
+
+      - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions
+        run: |
+          sudo bash << EOF
+              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
+              echo '{}' > /etc/docker/daemon.json
+              systemctl restart docker || journalctl --no-pager -n 500
+              systemctl status docker
+          EOF
+
+      # Default name for helm/kind-action kind clusters is "chart-testing"
+      - name: Create kind cluster with 5 nodes
+        uses: helm/kind-action@v1.8.0
+        with:
+          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
+          version: v0.14.0
+
+      - name: Preload previously built images onto kind cluster
+        run: kind load docker-image ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }} --name chart-testing
+
+      - name: Do not wait for an hour before detecting the rebootSentinel
+        run: |
+          sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds.yaml
+          sed -i 's/#\(.*\)--concurrency=1/\1--concurrency=2/g' kured-ds.yaml
+
+      - name: Install kured with kubectl
+        run: |
+          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml
+
+      - name: Ensure kured is ready
+        uses: nick-invision/retry@v2.9.0
+        with:
+          timeout_minutes: 10
+          max_attempts: 10
+          retry_wait_seconds: 60
+          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
+          command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
+
+      - name: Create reboot sentinel files
+        run: |
+          ./tests/kind/create-reboot-sentinels.sh
+
+      - name: Follow reboot until success
+        env:
+          DEBUG: true
+        run: |
+          ./tests/kind/follow-coordinated-reboot.sh

.github/workflows/on-tag.yaml

@@ -7,34 +7,91 @@ on:
   push:
     tags:
       - "*"
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
 jobs:
   tag-scan-and-push-final-image:
     name: "Build, scan, and publish tagged image"
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+      packages: write
     steps:
-      - uses: actions/checkout@master
-      - name: Find go version
-        run: |
-          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-          echo "::set-output name=version::${GO_VERSION}"
-        id: awk_gomod
+      - uses: actions/checkout@v4
       - name: Ensure go version
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: "${{ steps.awk_gomod.outputs.version }}"
-      - run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${GITHUB_REF#refs/tags/}" image
-
-      - uses: Azure/container-scan@v0
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: Find current tag version
+        run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+        id: tags
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Setup GoReleaser
+        run: make bootstrap-tools
+      - name: Build binaries
+        run: make kured-release-tag
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build single image for scan
+        uses: docker/build-push-action@v5
         with:
-          image-name: docker.io/${{ github.repository_owner }}/kured:${GITHUB_REF#refs/tags/}
+          context: .
+          platforms: linux/amd64
+          push: false
+          load: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
 
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@d43c1f16c00cfd3978dde6c07f4bbcf9eb6993ca
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME_WEAVEWORKSKUREDCI }}
-          password: ${{ secrets.DOCKERHUB_TOKEN_WEAVEWORKSKUREDCI }}
+          image-ref: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}'
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
 
-      - name: Publish image
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build release images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/arm64, linux/amd64, linux/arm/v7, linux/arm/v6, linux/386
+          push: true
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
+
+      - name: Generate SBOM
         run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="${GITHUB_REF#refs/tags/}" publish-image
+          .tmp/syft ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }} -o spdx > kured.sbom
+
+      - name: Sign and attest artifacts
+        run: |
+          .tmp/cosign sign -y -r ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
+
+          .tmp/cosign sign-blob -y --output-signature kured.sbom.sig kured.sbom
+
+          .tmp/cosign attest -y --type spdx --predicate kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
+          .tmp/cosign attach sbom --type spdx --sbom kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}

.github/workflows/periodics-daily.yaml

@@ -5,123 +5,76 @@ on:
   - cron: "30 1 * * *"
 
 jobs:
+  periodics-gotest:
+    name: Run go tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: run tests
+        run: go test -json ./... > test.json
+      - name: Annotate tests
+        if: always()
+        uses: guyarb/golang-test-annoations@v0.7.0
+        with:
+          test-results: test.json
+
   periodics-mark-stale:
     name: Mark stale issues and PRs
     runs-on: ubuntu-latest
     steps:
-    # Stale by default waits for 60 days before marking PR/issues as stale, and closes them after 7 days.
+    # Stale by default waits for 60 days before marking PR/issues as stale, and closes them after 21 days.
     # Do not expire the first issues that would allow the community to grow.
-    - uses: actions/stale@v3.0.14
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue was automatically considered stale due to lack of activity. Please update it and/or join our slack channels to promote it, before it automatically closes (in 7 days).'
         stale-pr-message: 'This PR was automatically considered stale due to lack of activity. Please refresh it and/or join our slack channels to highlight it, before it automatically closes (in 7 days).'
         stale-issue-label: 'no-issue-activity'
         stale-pr-label: 'no-pr-activity'
-        exempt-issue-labels: 'good-first-issue'
+        exempt-issue-labels: 'good first issue,keep'
+        days-before-close: 21
 
   check-docs-links:
     name: Check docs for incorrect links
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Link Checker
-      id: lc
-      uses: peter-evans/link-checker@v1
+      uses: lycheeverse/lychee-action@c3089c702fbb949e3f7a8122be0c33c017904f9b
+      env:
+        GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
       with:
-        args: -r *.md *.yaml */*/*.go -x .cluster.local
-    - name: Fail if there were link errors
-      run: exit ${{ steps.lc.outputs.exit_code }}
+        args: --verbose --no-progress '*.md' '*.yaml' '*/*/*.go' --exclude-link-local
+        fail: true
 
   vuln-scan:
     name: Build image and scan it against known vulnerabilities
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Find go version
-        run: |
-          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-          echo "::set-output name=version::${GO_VERSION}"
-        id: awk_gomod
+      - uses: actions/checkout@v4
       - name: Ensure go version
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: "${{ steps.awk_gomod.outputs.version }}"
-      - run: make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
-      - uses: Azure/container-scan@v0
-        with:
-          image-name: docker.io/${{ github.repository_owner }}/kured:${{ github.sha }}
-
-  deploy-helm:
-    name: Ensure a kubernetes change didn't break our code
-    runs-on: ubuntu-latest
-    # only build with oldest and newest supported, it should be good enough.
-    strategy:
-      matrix:
-        kubernetes:
-          - 1.18
-          - 1.19
-          - 1.20
-    steps:
-      - uses: actions/checkout@v2
-      - name: Find go version
-        run: |
-          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
-          echo "::set-output name=version::${GO_VERSION}"
-        id: awk_gomod
-      - name: Ensure go version
-        uses: actions/setup-go@v2
-        with:
-          go-version: "${{ steps.awk_gomod.outputs.version }}"
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Setup GoReleaser
+        run: make bootstrap-tools
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
       - name: Build artifacts
-        run: |
-          make DH_ORG="${{ github.repository_owner }}" VERSION="master" image
-          make DH_ORG="${{ github.repository_owner }}" VERSION="master" helm-chart
-
-      - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions"
-        run: |
-          sudo bash << EOF
-              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
-              echo '{}' > /etc/docker/daemon.json
-              systemctl restart docker || journalctl --no-pager -n 500
-              systemctl status docker
-          EOF
-
-      # Default name for helm/kind-action kind clusters is "chart-testing"
-      - name: Create 5 node kind cluster
-        uses: helm/kind-action@master
+        run: VERSION="${{ steps.tags.outputs.sha_short }}" make image
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@d43c1f16c00cfd3978dde6c07f4bbcf9eb6993ca
         with:
-          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
-
-      - name: Preload previously built images onto kind cluster
-        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:master --name chart-testing
-
-      - name: Deploy kured on default namespace with its helm chart
-        run: |
-          # Documented in official helm doc to live on the edge
-          curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
-          # Refresh bins
-          hash -r
-          helm install kured ./charts/kured/ --set configuration.period=1m
-          kubectl config set-context kind-chart-testing
-          kubectl get ds --all-namespaces
-          kubectl describe ds kured
-
-      - name: Ensure kured is ready
-        uses: nick-invision/retry@v2.4.0
-        with:
-          timeout_minutes: 10
-          max_attempts: 10
-          retry_wait_seconds: 60
-          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = 5
-          command: "kubectl get ds kured | grep -E 'kured.*5.*5.*5.*5.*5' "
-
-      - name: Create reboot sentinel files
-        run: |
-          ./tests/kind/create-reboot-sentinels.sh
-
-      - name: Follow reboot until success
-        env:
-          DEBUG: true
-        run: |
-          ./tests/kind/follow-coordinated-reboot.sh
+          image-ref: 'ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }}'
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'

.gitignore

@@ -1,3 +1,5 @@
 cmd/kured/kured
 vendor
 build
+dist
+.tmp

.goreleaser.yml

@@ -0,0 +1,32 @@
+project_name: kured
+before:
+  hooks:
+    - go mod tidy
+builds:
+  - main: ./cmd/kured
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+      - arm
+      - "386"
+    goarm:
+      - "6"
+      - "7"
+    ldflags:
+      - -s -w -X main.version={{ if .IsSnapshot }}{{ .ShortCommit }}{{ else }}{{ .Version }}{{ end }}
+    mod_timestamp: "{{ .CommitTimestamp }}"
+    flags:
+      - -trimpath
+
+snapshot:
+  name_template: "{{ .ShortCommit }}"
+
+release:
+  disable: true
+
+changelog:
+  skip: true

.lycheeignore

@@ -0,0 +1,6 @@
+app.fossa.com
+cluster.local
+hooks.slack.com
+localhost
+slack://
+teams://

CODE_OF_CONDUCT.md

@@ -0,0 +1,3 @@
+## Kured Community Code of Conduct
+
+Kured follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).

CONTRIBUTING.md

@@ -0,0 +1,241 @@
+# Developing `kured`
+
+We love contributions to `kured`, no matter if you are [helping out on
+Slack][slack], reporting or triaging [issues][issues] or contributing code
+to `kured`.
+
+In any case, it will make sense to familiarise yourself with the main
+[README][readme] to understand the different features and options, which is
+helpful for testing. The "building" section in particular makes sense if
+you are planning to contribute code.
+
+[slack]: README.md#getting-help
+[issues]: https://github.com/kubereboot/kured/issues
+[readme]: README.md
+
+## Certificate of Origin
+
+By contributing to this project you agree to the Developer Certificate of
+Origin (DCO). This document was created by the Linux Kernel community and is a
+simple statement that you, as a contributor, have the legal right to make the
+contribution.
+
+We require all commits to be signed. By signing off with your signature, you
+certify that you wrote the patch or otherwise have the right to contribute the
+material by the rules of the [DCO](DCO):
+
+`Signed-off-by: Jane Doe <jane.doe@example.com>`
+
+The signature must contain your real name
+(sorry, no pseudonyms or anonymous contributions)
+If your `user.name` and `user.email` are configured in your Git config,
+you can sign your commit automatically with `git commit -s`.
+
+## Kured Repositories
+
+All Kured repositories are kept under <https://github.com/kubereboot>. To find the code and work on the individual pieces that make Kured, here is our overview:
+
+| Repositories                            | Contents                  |
+| --------------------------------------- | ------------------------- |
+| <https://github.com/kubereboot/kured>   | Kured operator itself     |
+| <https://github.com/kubereboot/charts>  | Helm chart                |
+| <https://github.com/kubereboot/website> | website and documentation |
+
+## Regular development activities
+
+### Prepare environment
+
+Please run `make bootstrap-tools` once on a fresh repository clone to download several needed tools, e.g. GoReleaser.
+
+### Updating k8s support
+
+Whenever we want to update e.g. the `kubectl` or `client-go` dependencies,
+some RBAC changes might be necessary too.
+
+This is what it took to support Kubernetes 1.14:
+<https://github.com/kubereboot/kured/pull/75>
+
+That the process can be more involved based on kubernetes changes.
+For example, k8s 1.10 changes to apps triggered the following commits:
+
+b3f9ddf: Bump client-go for optimum k8s 1.10 compatibility
+bc3f28d: Move deployment manifest to apps/v1
+908998a: Update RBAC permissions for kubectl v1.10.3
+efbb0c3: Document version compatibility in release notes
+5731b98: Add warning to Dockerfile re: upgrading kubectl
+
+Search the git log for inspiration for your cases.
+
+Please update our `.github/workflows` with the new k8s images, starting by
+the creation of a `.github/kind-cluster-<version>.yaml`, then updating
+our workflows with the new versions.
+
+Once you updated everything, make sure you update the support matrix on
+the main [README][readme] as well.
+
+### Updating other dependencies
+
+Dependabot proposes changes in our go.mod/go.sum.
+Some of those changes are covered by CI testing, some are not.
+
+Please make sure to test those not covered by CI (mostly the integration
+with other tools) manually before merging.
+
+### Review periodic jobs
+
+We run periodic jobs (see also Automated testing section of this documentation).
+Those should be monitored for failures.
+
+If a failure happen in periodics, something terribly wrong must have happened
+(or github is failing at the creation of a kind cluster). Please monitor those
+failures carefully.
+
+### Introducing new features
+
+When you introduce a new feature, the kured team expects you to have tested
+your change thoroughly. If possible, include all the necessary testing in your change.
+
+If your change involves a user facing change (change in flags of kured for example),
+please include expose your new feature in our default manifest (`kured-ds.yaml`),
+as a comment.
+
+Our release manifests and helm charts are our stable interfaces.
+Any user facing changes will therefore have to wait for a release before being
+exposed to our users.
+
+This also means that when you expose a new feature, you should create another PR
+for your changes in <https://github.com/kubereboot/charts> to make your feature
+available at the next kured version for helm users.
+
+In the charts PR, you can directly bump the appVersion to the next minor version
+(you are introducing a new feature, which requires a bump of the minor number.
+For example, if current appVersion is 1.6.x, make sure you update your appVersion
+to 1.7.0). It allows us to have an easy view of what we land each release.
+
+Do not hesitate to increase the test coverage for your feature, whether it's unit
+testing to full functional testing (even using helm charts)
+
+### Increasing test coverage
+
+We are welcoming any change to increase our test coverage.
+See also our github issues for the label `testing`.
+
+## Automated testing
+
+Our CI is covered by github actions.
+You can see their contents in .github/workflows.
+
+We currently run:
+
+- go tests and lint
+- `shellcheck`
+- a check for dead links in our docs
+- a security check against our base image (alpine)
+- a deep functional test using our manifests on all supported k8s versions
+
+To test your code manually, follow the section Manual testing.
+
+## Manual (release) testing
+
+Before `kured` is released, we want to make sure it still works fine on the
+previous, current and next minor version of Kubernetes (with respect to the
+`client-go` & `kubectl` dependencies in use). For local testing e.g.
+`minikube` or `kind` can be sufficient. This will allow you to catch issues
+that might not have been tested in our CI, like integration with other tools,
+or your specific use case.
+
+Deploy kured in your test scenario, make sure you pass the right `image`,
+update the e.g. `period` and `reboot-days` options, so you get immediate
+results, if you login to a node and run:
+
+```console
+sudo touch /var/run/reboot-required
+```
+
+### Example of golang testing
+
+Please run `make test`. You should have `golint` installed.
+
+### Example of testing with `minikube`
+
+A test-run with `minikube` could look like this:
+
+```console
+# start minikube
+minikube start --driver=kvm2 --kubernetes-version <k8s-release>
+
+# build kured image and publish to registry accessible by minikube
+make image minikube-publish
+
+# edit kured-ds.yaml to
+#   - point to new image
+#   - change e.g. period and reboot-days option for immediate results
+
+minikube kubectl -- apply -f kured-rbac.yaml
+minikube kubectl -- apply -f kured-ds.yaml
+minikube kubectl -- logs daemonset.apps/kured -n kube-system -f
+
+# In separate terminal
+minikube ssh
+ sudo touch /var/run/reboot-required
+minikube logs -f
+```
+
+Now check for the 'Commanding reboot' message and minikube going down.
+
+Unfortunately as of today, you are going to run into
+<https://github.com/kubernetes/minikube/issues/2874>. This means that
+minikube won't come back easily. You will need to start minikube again.
+Then you can check for the lock release.
+
+### Example of testing with `kind`
+
+A test-run with `kind` could look like this:
+
+```console
+# create kind cluster
+kind create cluster --config .github/kind-cluster-<k8s-version>.yaml
+
+# create reboot required files on pre-defined kind nodes
+./tests/kind/create-reboot-sentinels.sh
+
+# check if reboot is working fine
+./tests/kind/follow-coordinated-reboot.sh
+
+```
+
+## Publishing a new kured release
+
+### Prepare Documentation
+
+Check that [compatibility matrix](https://kured.dev/docs/installation/) is updated
+to the new version you want to release.
+
+### Create a tag on the repo
+
+Before going further, we should freeze the code for a release, by
+tagging the code. The Github-Action should start a new job and push
+the new image to the registry.
+
+### Create the combined manifest
+
+Now create the `kured-<release>-dockerhub.yaml` for e.g. `1.3.0`:
+
+```sh
+VERSION=1.3.0
+MANIFEST="kured-$VERSION-dockerhub.yaml"
+make DH_ORG="kubereboot" VERSION="${VERSION}" manifest
+cat kured-rbac.yaml > "$MANIFEST"
+cat kured-ds.yaml >> "$MANIFEST"
+```
+
+### Publish release artifacts
+
+Now you can head to the Github UI, use the version number as tag and upload the
+`kured-<release>-dockerhub.yaml` file.
+
+Please describe what's new and noteworthy in the release notes, list the PRs
+that landed and give a shout-out to everyone who contributed.
+
+Please also note down on which releases the upcoming `kured` release was
+tested on. (Check old release notes if you're unsure.)

DCO

@@ -0,0 +1,36 @@
+Developer Certificate of Origin
+Version 1.1
+
+Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+660 York Street, Suite 102,
+San Francisco, CA 94110 USA
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+
+Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.

DEVELOPMENT.md

@@ -1,157 +1 @@
-# Developing `kured`
-
-We love contributions to `kured`, no matter if you are [helping out on
-Slack][slack], reporting or triaging [issues][issues] or contributing code
-to `kured`.
-
-In any case, it will make sense to familiarise yourself with the main
-[README][readme] to understand the different features and options, which is
-helpful for testing. The "building" section in particular makes sense if
-you are planning to contribute code.
-
-[slack]: README.md#getting-help
-[issues]: https://github.com/weaveworks/kured/issues
-[readme]: README.md
-
-## Updating k8s support
-
-Whenever we want to update e.g. the `kubectl` or `client-go` dependencies,
-some RBAC changes might be necessary too.
-
-This is what it took to support Kubernetes 1.14:
-<https://github.com/weaveworks/kured/pull/75>
-
-That the process can be more involved that that can be seen in
-<https://github.com/weaveworks/kured/commits/support-k8s-1.10>
-
-Once you updated everything, make sure you update the support matrix on
-the main [README][readme] as well.
-
-## Release testing
-
-Before `kured` is released, we want to make sure it still works fine on the
-previous, current and next minor version of Kubernetes (with respect to the
-embedded `client-go` & `kubectl`). For local testing e.g. `minikube` or
-`kind` can be sufficient.
-
-Deploy kured in your test scenario, make sure you pass the right `image`,
-update the e.g. `period` and `reboot-days` options, so you get immediate
-results, if you login to a node and run:
-
-```console
-sudo touch /var/run/reboot-required
-```
-
-### Testing with `minikube`
-
-A test-run with `minikube` could look like this:
-
-```console
-# start minikube
-minikube start --vm-driver kvm2 --kubernetes-version <k8s-release>
-
-# build kured image and publish to registry accessible by minikube
-make image minikube-publish
-
-# edit kured-ds.yaml to
-#   - point to new image
-#   - change e.g. period and reboot-days option for immediate results
-
-minikube kubectl -- apply -f kured-rbac.yaml
-minikube kubectl -- apply -f kured-ds.yaml
-minikube kubectl -- logs daemonset.apps/kured -n kube-system -f
-
-# Alternatively use helm to install the chart
-# edit values-local.yaml to change any chart parameters
-helm install kured ./charts/kured --namespace kube-system -f ./charts/kured/values.minikube.yaml
-
-# In separate terminal
-minikube ssh
- sudo touch /var/run/reboot-required
-minikube logs -f
-```
-
-Now check for the 'Commanding reboot' message and minikube going down.
-
-Unfortunately as of today, you are going to run into
-<https://github.com/kubernetes/minikube/issues/2874>. This means that
-minikube won't come back easily. You will need to start minikube again.
-Then you can check for the lock release.
-
-If all the tests ran well, kured maintainers can reach out to the Weaveworks
-team to get an upcoming `kured` release tested in the Dev environment for
-real life testing.
-
-### Testing with `kind`
-
-A test-run with `kind` could look like this:
-
-```console
-# create kind cluster
-kind create cluster --config .github/kind-cluster.yaml
-
-# create reboot required files on pre-defined kind nodes
-./tests/create-reboot-sentinels.sh
-
-# check if reboot is working fine
-./tests/kind/follow-coordinated-reboot.sh
-
-```
-
-## Publishing a new kured release
-
-### Prepare Documentation
-Check that `README.md` has an updated compatibility matrix and that the
-url in the `kubectl` incantation (under "Installation") is updated to the
-new version you want to release.
-
-### Create a tag on the repo and publish the image
-
-Before going further, we should freeze the code for a release, by
-tagging the code, and publishing its immutable artifact: the kured
-docker image.
-
-```sh
-make DH_ORG="weaveworks" VERSION="1.3.0" image
-```
-
-Then docker push the image. In the future, that might be automatically
-done when creating a tag on the repository, with the help of github
-actions.
-
-### Create the combined manifest
-
-
-Now create the `kured-<release>-dockerhub.yaml` for e.g. `1.3.0`:
-
-```sh
-VERSION=1.3.0
-MANIFEST="kured-$VERSION-dockerhub.yaml"
-make DH_ORG="weaveworks" VERSION="${VERSION}" manifest
-cat kured-rbac.yaml > "$MANIFEST"
-cat kured-ds.yaml >> "$MANIFEST"
-```
-### Publish release artifacts
-
-Now you can head to the Github UI, use the version number as tag and upload the
-`kured-<release>-dockerhub.yaml` file.
-
-Please describe what's new and noteworthy in the release notes, list the PRs
-that landed and give a shout-out to everyone who contributed.
-
-Please also note down on which releases the upcoming `kured` release was
-tested on. (Check old release notes if you're unsure.)
-
-### Update the Helm chart
-
-You can automatically bump the helm chart's application version
-with the latest image tag by running:
-
-```sh
-make DH_ORG="weaveworks" VERSION="1.3.0" helm-chart
-```
-
-A change in the helm chart requires a bump of the `version`
-in `charts/kured/Chart.yaml` (following the versioning rules).
-Update it, and issue a PR. Upon merge, that PR will automatically
-publish the chart to the gh-pages branch.
+This file was moved to [CONTRIBUTING.md](CONTRIBUTING.md).

Dockerfile

@@ -0,0 +1,25 @@
+FROM --platform=$TARGETPLATFORM alpine:3.19.0 as bin
+
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+COPY dist/ /dist
+RUN set -ex \
+  && case "${TARGETARCH}" in \
+      amd64) \
+          SUFFIX="_v1" \
+          ;; \
+      arm) \
+          SUFFIX="_${TARGETVARIANT:1}" \
+          ;; \
+      *) \
+          SUFFIX="" \
+          ;; \
+    esac \
+  && cp /dist/kured_${TARGETOS}_${TARGETARCH}${SUFFIX}/kured /dist/kured;
+
+FROM --platform=$TARGETPLATFORM alpine:3.19.0
+RUN apk update --no-cache && apk upgrade --no-cache && apk add --no-cache ca-certificates tzdata
+COPY --from=bin /dist/kured /usr/bin/kured
+ENTRYPOINT ["/usr/bin/kured"]

GOVERNANCE.md

@@ -0,0 +1,112 @@
+# Project Governance
+
+- [Values](#values)
+- [Maintainers](#maintainers)
+- [Becoming a Maintainer](#becoming-a-maintainer)
+- [Meetings](#meetings)
+- [Code of Conduct Enforcement](#code-of-conduct)
+- [Voting](#voting)
+
+## Values
+
+The Kured project and its leadership embrace the following values:
+
+- Openness: Communication and decision-making happens in the open and is discoverable for future
+  reference. As much as possible, all discussions and work take place in public
+  forums and open repositories.
+
+- Fairness: All stakeholders have the opportunity to provide feedback and submit
+  contributions, which will be considered on their merits.
+
+- Community over Product or Company: Sustaining and growing our community takes
+  priority over shipping code or sponsors' organizational goals.  Each
+  contributor participates in the project as an individual.
+
+- Inclusivity: We innovate through different perspectives and skill sets, which
+  can only be accomplished in a welcoming and respectful environment.
+
+- Participation: Responsibilities within the project are earned through
+  participation, and there is a clear path up the contributor ladder into leadership
+  positions.
+
+- Consensus: Whether or not wider input is required, the Kured community believes that
+  the best decisions are reached through Consensus
+  <https://en.wikipedia.org/wiki/Consensus_decision-making>.
+
+## Maintainers
+
+Kured Maintainers have write access to the [project GitHub
+organisation](https://github.com/kubereboot). They can merge their own patches or patches
+from others. The current maintainers can be found in [MAINTAINERS][maintainers-file].
+Maintainers collectively manage the project's resources and contributors.
+
+This privilege is granted with some expectation of responsibility: maintainers
+are people who care about the Kured project and want to help it grow and
+improve. A maintainer is not just someone who can make changes, but someone who
+has demonstrated their ability to collaborate with the team, get the most
+knowledgeable people to review code and docs, contribute high-quality code, and
+follow through to fix issues (in code or tests).
+
+A maintainer is a contributor to the project's success and a citizen helping
+the project succeed.
+
+## Becoming a Maintainer
+
+To become a Maintainer you need to demonstrate the following:
+
+- commitment to the project:
+  - participate in discussions, contributions, code and documentation reviews
+      for 3 months or more and participate in Slack discussions and meetings
+      if possible,
+  - perform reviews for 5 non-trivial pull requests,
+  - contribute 5 non-trivial pull requests and have them merged,
+- ability to write quality code and/or documentation,
+- ability to collaborate with the team,
+- understanding of how the team works (policies, processes for testing and code review, etc),
+- understanding of the project's code base and coding and documentation style.
+
+We realise that everybody brings different abilities and qualities to the team, that's
+why we are willing to change the rules somewhat depending on the circumstances.
+
+A new Maintainer can apply by proposing a PR to the [MAINTAINERS
+file][maintainers-file]. A simple majority vote of existing Maintainers
+approves the application.
+
+Maintainers who are selected will be granted the necessary GitHub rights,
+and invited to the [private maintainer mailing list][private-list].
+
+## Meetings
+
+Time zones permitting, Maintainers are expected to participate in the public
+developer meeting, details can be found [here][meeting-agenda].
+
+Maintainers will also have closed meetings in order to discuss security reports
+or Code of Conduct violations.  Such meetings should be scheduled by any
+Maintainer on receipt of a security issue or CoC report.  All current Maintainers
+must be invited to such closed meetings, except for any Maintainer who is
+accused of a CoC violation.
+
+## Code of Conduct
+
+[Code of Conduct](./CODE_OF_CONDUCT.md) violations by community members will
+be discussed and resolved on the [private Maintainer mailing list][private-list].
+If the reported CoC violator is a Maintainer, the Maintainers will instead
+designate two Maintainers to work with CNCF staff in resolving the report.
+
+## Voting
+
+While most business in Kured is conducted by "lazy consensus", periodically
+the Maintainers may need to vote on specific actions or changes.
+A vote can be taken in [kured issues labeled 'decision'][decision-issues] or
+[the private Maintainer mailing list][private-list] for security or conduct
+matters. Votes may also be taken at [the developer meeting][meeting-agenda].
+Any Maintainer may demand a vote be taken.
+
+Most votes require a simple majority of all Maintainers to succeed. Maintainers
+can be removed by a 2/3 majority vote of all Maintainers, and changes to this
+Governance require a 2/3 vote of all Maintainers.
+
+[maintainers-file]: ./MAINTAINERS
+[private-list]: cncf-kured-maintainers@lists.cncf.io
+[meeting-agenda]: https://docs.google.com/document/d/1AWT8YDdqZY-Se6Y1oAlwtujWLVpNVK2M_F_Vfqw06aI/edit
+[decision-issues]: https://github.com/kubereboot/kured/labels/decision

MAINTAINERS

@@ -0,0 +1,5 @@
+Christian Hopf <christian.kotzbauer@gmail.com> (@ckotzbauer)
+Daniel Holbach <daniel.holbach@gmail.com> (@dholbach)
+Hidde Beydals <hidde@weave.works> (@hiddeco)
+Jack Francis <jackfrancis@gmail.com> (@jackfrancis)
+Jean-Philippe Evrard <open-source@a.spamming.party> (@evrardjp)

Makefile

@@ -1,45 +1,54 @@
 .DEFAULT: all
-.PHONY: all clean image publish-image minikube-publish manifest helm-chart
+.PHONY: all clean image minikube-publish manifest test kured-all
 
-DH_ORG=weaveworks
-VERSION=$(shell git symbolic-ref --short HEAD)-$(shell git rev-parse --short HEAD)
+TEMPDIR=./.tmp
+GORELEASER_CMD=$(TEMPDIR)/goreleaser
+DH_ORG=kubereboot
+VERSION=$(shell git rev-parse --short HEAD)
 SUDO=$(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 
 all: image
 
+$(TEMPDIR):
+	mkdir -p $(TEMPDIR)
+
+.PHONY: bootstrap-tools
+bootstrap-tools: $(TEMPDIR)
+	VERSION=v1.19.2 TMPDIR=.tmp bash .github/scripts/goreleaser-install.sh
+	curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b .tmp v0.86.1
+	curl -sSfL https://github.com/sigstore/cosign/releases/download/v2.1.1/cosign-linux-amd64 -o .tmp/cosign
+	chmod +x .tmp/goreleaser .tmp/cosign .tmp/syft
+
 clean:
-	rm -f cmd/kured/kured
-	rm -rf ./build
+	rm -rf ./dist
 
-godeps=$(shell go list -f '{{join .Deps "\n"}}' $1 | grep -v /vendor/ | xargs go list -f '{{if not .Standard}}{{ $$dep := . }}{{range .GoFiles}}{{$$dep.Dir}}/{{.}} {{end}}{{end}}')
+kured:
+	$(GORELEASER_CMD) build --clean --single-target --snapshot
 
-DEPS=$(call godeps,./cmd/kured)
+kured-all:
+	$(GORELEASER_CMD) build --clean --snapshot
 
-cmd/kured/kured: $(DEPS)
-cmd/kured/kured: cmd/kured/*.go
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X main.version=$(VERSION)" -o $@ cmd/kured/*.go
+kured-release-tag:
+	$(GORELEASER_CMD) release --clean
 
-build/.image.done: cmd/kured/Dockerfile cmd/kured/kured
-	mkdir -p build
-	cp $^ build
-	$(SUDO) docker build -t docker.io/$(DH_ORG)/kured -f build/Dockerfile ./build
-	$(SUDO) docker tag docker.io/$(DH_ORG)/kured docker.io/$(DH_ORG)/kured:$(VERSION)
-	touch $@
+kured-release-snapshot:
+	$(GORELEASER_CMD) release --clean --snapshot
 
-image: build/.image.done
-
-publish-image: image
-	$(SUDO) docker push docker.io/$(DH_ORG)/kured:$(VERSION)
+image: kured
+	$(SUDO) docker buildx build --load -t ghcr.io/$(DH_ORG)/kured:$(VERSION) .
 
 minikube-publish: image
-	$(SUDO) docker save docker.io/$(DH_ORG)/kured | (eval $$(minikube docker-env) && docker load)
+	$(SUDO) docker save ghcr.io/$(DH_ORG)/kured | (eval $$(minikube docker-env) && docker load)
 
 manifest:
-	sed -i "s#image: docker.io/.*kured.*#image: docker.io/$(DH_ORG)/kured:$(VERSION)#g" kured-ds.yaml
+	sed -i "s#image: ghcr.io/.*kured.*#image: ghcr.io/$(DH_ORG)/kured:$(VERSION)#g" kured-ds.yaml
+	sed -i "s#image: ghcr.io/.*kured.*#image: ghcr.io/$(DH_ORG)/kured:$(VERSION)#g" kured-ds-signal.yaml
 	echo "Please generate combined manifest if necessary"
 
-helm-chart:
-	sed -i "s#repository:.*/kured#repository: $(DH_ORG)/kured#g" charts/kured/values.yaml
-	sed -i "s#tag:.*#tag: $(VERSION)#g" charts/kured/values.yaml
-	sed -i "s#appVersion:.*#appVersion: \"$(VERSION)\"#g" charts/kured/Chart.yaml
-	echo "Please bump version in charts/kured/Chart.yaml"
+test:
+	echo "Running go tests"
+	go test ./...
+	echo "Running golint on pkg"
+	golint ./pkg/...
+	echo "Running golint on cmd"
+	golint ./cmd/...

README.md

@@ -1,27 +1,17 @@
-
 # kured - Kubernetes Reboot Daemon
 
-<img src="https://github.com/weaveworks/kured/raw/master/img/logo.png" align="right"/>
+[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/kured)](https://artifacthub.io/packages/helm/kured/kured)
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkubereboot%2Fkured.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkubereboot%2Fkured?ref=badge_shield)
+[![CLOMonitor](https://img.shields.io/endpoint?url=https://clomonitor.io/api/projects/cncf/kured/badge)](https://clomonitor.io/projects/cncf/kured)
 
-* [Introduction](#introduction)
-* [Kubernetes & OS Compatibility](#kubernetes-&-os-compatibility)
-* [Installation](#installation)
-* [Configuration](#configuration)
-  * [Reboot Sentinel File & Period](#reboot-sentinel-file-&-period)
-  * [Setting a schedule](#setting-a-schedule)
-  * [Blocking Reboots via Alerts](#blocking-reboots-via-alerts)
-  * [Blocking Reboots via Pods](#blocking-reboots-via-pods)
-  * [Prometheus Metrics](#prometheus-metrics)
-  * [Slack Notifications](#slack-notifications)
-  * [Overriding Lock Configuration](#overriding-lock-configuration)
-* [Operation](#operation)
-  * [Testing](#testing)
-  * [Disabling Reboots](#disabling-reboots)
-  * [Manual Unlock](#manual-unlock)
-  * [Automatic Unlock](#automatic-unlock)
-* [Building](#building)
-* [Frequently Asked/Anticipated Questions](#frequently-askedanticipated-questions)
-* [Getting Help](#getting-help)
+<img src="https://github.com/kubereboot/website/raw/main/static/img/kured.png" width="200" align="right"/>
+
+- [kured - Kubernetes Reboot Daemon](#kured---kubernetes-reboot-daemon)
+  - [Introduction](#introduction)
+  - [Documentation](#documentation)
+  - [Getting Help](#getting-help)
+  - [Trademarks](#trademarks)
+  - [License](#license)
 
 ## Introduction
 
@@ -29,303 +19,48 @@ Kured (KUbernetes REboot Daemon) is a Kubernetes daemonset that
 performs safe automatic node reboots when the need to do so is
 indicated by the package management system of the underlying OS.
 
-* Watches for the presence of a reboot sentinel e.g. `/var/run/reboot-required`
-* Utilises a lock in the API server to ensure only one node reboots at
+- Watches for the presence of a reboot sentinel file e.g. `/var/run/reboot-required`
+  or the successful run of a sentinel command.
+- Utilises a lock in the API server to ensure only one node reboots at
   a time
-* Optionally defers reboots in the presence of active Prometheus alerts or selected pods
-* Cordons & drains worker nodes before reboot, uncordoning them after
+- Optionally defers reboots in the presence of active Prometheus alerts or selected pods
+- Cordons & drains worker nodes before reboot, uncordoning them after
 
-## Kubernetes & OS Compatibility
+## Documentation
 
-The daemon image contains versions of `k8s.io/client-go` and
-`k8s.io/kubectl` (the binary of `kubectl` in older releases) for the purposes of
-maintaining the lock and draining worker nodes. Kubernetes aims to provide
-forwards and backwards compatibility of one minor version between client and
-server:
+Find all our docs on <https://kured.dev>:
 
-| kured  | kubectl | k8s.io/client-go | k8s.io/apimachinery | expected kubernetes compatibility |
-|--------|---------|------------------|---------------------|-----------------------------------|
-| master | 1.19.4  | v0.19.4          | v0.19.4             | 1.18.x, 1.19.x, 1.20.x            |
-| 1.6.0  | 1.19.4  | v0.19.4          | v0.19.4             | 1.18.x, 1.19.x, 1.20.x            |
-| 1.5.1  | 1.18.8  | v0.18.8          | v0.18.8             | 1.17.x, 1.18.x, 1.19.x            |
-| 1.4.4  | 1.17.7  | v0.17.0          | v0.17.0             | 1.16.x, 1.17.x, 1.18.x            |
-| 1.3.0  | 1.15.10 | v12.0.0          | release-1.15        | 1.15.x, 1.16.x, 1.17.x            |
-| 1.2.0  | 1.13.6  | v10.0.0          | release-1.13        | 1.12.x, 1.13.x, 1.14.x            |
-| 1.1.0  | 1.12.1  | v9.0.0           | release-1.12        | 1.11.x, 1.12.x, 1.13.x            |
-| 1.0.0  | 1.7.6   | v4.0.0           | release-1.7         | 1.6.x, 1.7.x, 1.8.x               |
+- [All Kured Documentation](https://kured.dev/docs/)
+- [Installing Kured](https://kured.dev/docs/installation/)
+- [Configuring Kured](https://kured.dev/docs/configuration/)
+- [Operating Kured](https://kured.dev/docs/operation/)
+- [Developing Kured](https://kured.dev/docs/development/)
 
-See the [release notes](https://github.com/weaveworks/kured/releases)
-for specific version compatibility information, including which
-combination have been formally tested.
-
-Versions >=1.1.0 enter the host mount namespace to invoke
-`systemctl reboot`, so should work on any systemd distribution.
-
-## Installation
-
-To obtain a default installation without Prometheus alerting interlock
-or Slack notifications:
-
-```console
-latest=$(curl -s https://api.github.com/repos/weaveworks/kured/releases | jq -r .[0].tag_name)
-kubectl apply -f "https://github.com/weaveworks/kured/releases/download/$latest/kured-$latest-dockerhub.yaml"
-```
-
-If you want to customise the installation, download the manifest and
-edit it in accordance with the following section before application.
-
-## Configuration
-
-The following arguments can be passed to kured via the daemonset pod template:
-
-```console
-Flags:
-      --lock-ttl time                       force clean annotation after this ammount of time (default 0, disabled)
-      --alert-filter-regexp regexp.Regexp   alert names to ignore when checking for active alerts
-      --blocking-pod-selector stringArray   label selector identifying pods whose presence should prevent reboots
-      --ds-name string                      name of daemonset on which to place lock (default "kured")
-      --ds-namespace string                 namespace containing daemonset on which to place lock (default "kube-system")
-      --end-time string                     only reboot before this time of day (default "23:59")
-  -h, --help                                help for kured
-      --lock-annotation string              annotation in which to record locking node (default "weave.works/kured-node-lock")
-      --period duration                     reboot check period (default 1h0m0s)
-      --prometheus-url string               Prometheus instance to probe for active alerts
-      --reboot-days strings                 only reboot on these days (default [su,mo,tu,we,th,fr,sa])
-      --reboot-sentinel string              path to file whose existence signals need to reboot (default "/var/run/reboot-required")
-      --slack-channel string                slack channel for reboot notfications
-      --slack-hook-url string               slack hook URL for reboot notfications
-      --slack-username string               slack username for reboot notfications (default "kured")
-      --message-template-drain string       message template used to notify about a node being drained (default "Draining node %s")
-      --message-template-reboot string      message template used to notify about a node being rebooted (default "Rebooting node %s")
-      --start-time string                   only reboot after this time of day (default "0:00")
-      --time-zone string                    use this timezone to calculate allowed reboot time (default "UTC")
-```
-
-### Reboot Sentinel File & Period
-
-By default kured checks for the existence of
-`/var/run/reboot-required` every sixty minutes; you can override these
-values with `--reboot-sentinel` and `--period`. Each replica of the
-daemon uses a random offset derived from the period on startup so that
-nodes don't all contend for the lock simultaneously.
-
-### Setting a schedule
-
-By default, kured will reboot any time it detects the sentinel, but this
-may cause reboots during odd hours.  While service disruption does not
-normally occur, anything is possible and operators may want to restrict
-reboots to predictable schedules.  Use `--reboot-days`, `--start-time`,
-`--end-time`, and `--time-zone` to set a schedule.  For example, business
-hours on the west coast USA can be specified with:
-
-```console
-  --reboot-days=mon,tue,wed,thu,fri
-  --start-time=9am
-  --end-time=5pm
-  --time-zone=America/Los_Angeles
-```
-
-Times can be formatted in numerous ways, including `5pm`, `5:00pm` `17:00`,
-and `17`.  `--time-zone` represents a Go `time.Location`, and can be `UTC`,
-`Local`, or any entry in the standard Linux tz database.
-
-Note that when using smaller time windows, you should consider shortening
-the sentinel check period (`--period`).
-
-### Blocking Reboots via Alerts
-
-You may find it desirable to block automatic node reboots when there
-are active alerts - you can do so by providing the URL of your
-Prometheus server:
-
-```console
---prometheus-url=http://prometheus.monitoring.svc.cluster.local
-```
-
-By default the presence of *any* active (pending or firing) alerts
-will block reboots, however you can ignore specific alerts:
-
-```console
---alert-filter-regexp=^(RebootRequired|AnotherBenignAlert|...$
-```
-
-See the section on Prometheus metrics for an important application of this
-filter.
-
-### Blocking Reboots via Pods
-
-You can also block reboots of an _individual node_ when specific pods
-are scheduled on it:
-
-```console
---blocking-pod-selector=runtime=long,cost=expensive
-```
-
-Since label selector strings use commas to express logical 'and', you can
-specify this parameter multiple times for 'or':
-
-```console
---blocking-pod-selector=runtime=long,cost=expensive
---blocking-pod-selector=name=temperamental
-```
-
-In this case, the presence of either an (appropriately labelled) expensive long
-running job or a known temperamental pod on a node will stop it rebooting.
-
-> Try not to abuse this mechanism - it's better to strive for
-> restartability where possible. If you do use it, make sure you set
-> up a RebootRequired alert as described in the next section so that
-> you can intervene manually if reboots are blocked for too long.
-
-### Prometheus Metrics
-
-Each kured pod exposes a single gauge metric (`:8080/metrics`) that
-indicates the presence of the sentinel file:
-
-```console
-# HELP kured_reboot_required OS requires reboot due to software updates.
-# TYPE kured_reboot_required gauge
-kured_reboot_required{node="ip-xxx-xxx-xxx-xxx.ec2.internal"} 0
-```
-
-The purpose of this metric is to power an alert which will summon an
-operator if the cluster cannot reboot itself automatically for a
-prolonged period:
-
-```console
-# Alert if a reboot is required for any machines. Acts as a failsafe for the
-# reboot daemon, which will not reboot nodes if there are pending alerts save
-# this one.
-ALERT RebootRequired
-  IF          max(kured_reboot_required) != 0
-  FOR         24h
-  LABELS      { severity="warning" }
-  ANNOTATIONS {
-    summary = "Machine(s) require being rebooted, and the reboot daemon has failed to do so for 24 hours",
-    impact = "Cluster nodes more vulnerable to security exploits. Eventually, no disk space left.",
-    description = "Machine(s) require being rebooted, probably due to kernel update.",
-  }
-```
-
-If you choose to employ such an alert and have configured kured to
-probe for active alerts before rebooting, be sure to specify
-`--alert-filter-regexp=^RebootRequired$` to avoid deadlock!
-
-### Slack Notifications
-
-If you specify a Slack hook via `--slack-hook-url`, kured will notify
-you immediately prior to rebooting a node:
-
-![Notification](img/slack-notification.png)
-
-We recommend setting `--slack-username` to be the name of the
-environment, e.g. `dev` or `prod`.
-
-Alternatively you can use the `--message-template-drain` and `--message-template-reboot` to customize the text of the message, e.g.
-```
---message-template-drain="Draining node %s part of *my-cluster* in region *xyz*"
-```
-
-### Overriding Lock Configuration
-
-The `--ds-name` and `--ds-namespace` arguments should match the name and
-namespace of the daemonset used to deploy the reboot daemon - the locking is
-implemented by means of an annotation on this resource. The defaults match
-the daemonset YAML provided in the repository.
-
-Similarly `--lock-annotation` can be used to change the name of the
-annotation kured will use to store the lock, but the default is almost
-certainly safe.
-
-## Operation
-
-The example commands in this section assume that you have not
-overriden the default lock annotation, daemonset name or namespace;
-if you have, you will have to adjust the commands accordingly.
-
-### Testing
-
-You can test your configuration by provoking a reboot on a node:
-
-```console
-sudo touch /var/run/reboot-required
-```
-
-### Disabling Reboots
-
-If you need to temporarily stop kured from rebooting any nodes, you
-can take the lock manually:
-
-```console
-kubectl -n kube-system annotate ds kured weave.works/kured-node-lock='{"nodeID":"manual"}'
-```
-
-Don't forget to release it afterwards!
-
-### Manual Unlock
-
-In exceptional circumstances, such as a node experiencing a permanent
-failure whilst rebooting, manual intervention may be required to
-remove the cluster lock:
-
-```console
-kubectl -n kube-system annotate ds kured weave.works/kured-node-lock-
-```
-
-> NB the `-` at the end of the command is important - it instructs
-> `kubectl` to remove that annotation entirely.
-
-### Automatic Unlock
-
-In exceptional circumstances (especially when used with cluster-autoscaler) a node
-which holds lock might be killed thus annotation will stay there for ever.
-
-Using `--lock-ttl=30m` will allow other nodes to take over if TTL has expired (in this case 30min) and continue reboot process.
-
-## Building
-
-Kured now uses [Go
-Modules](https://github.com/golang/go/wiki/Modules), so build
-instructions vary depending on where you have checked out the
-repository:
-
-**Building outside $GOPATH:**
-
-```console
-make
-```
-
-**Building inside $GOPATH:**
-
-```console
-GO111MODULE=on make
-```
-
-You can find the current preferred version of Golang in the [go.mod file](go.mod).
-
-If you are interested in contributing code to kured, please take a look at
-our [development][development] docs.
-
-[development]: DEVELOPMENT.md
-
-## Frequently Asked/Anticipated Questions
-
-### Why is there no `latest` tag on Docker Hub?
-
-Use of `latest` for production deployments is bad practice - see
-[here](https://kubernetes.io/docs/concepts/configuration/overview) for
-details. The manifest on `master` refers to `latest` for local
-development testing with minikube only; for production use choose a
-versioned manifest from the [release page](https://github.com/weaveworks/kured/releases/).
+And there's much more!
 
 ## Getting Help
 
 If you have any questions about, feedback for or problems with `kured`:
 
-* Invite yourself to the <a href="https://slack.weave.works/" target="_blank">Weave Users Slack</a>.
-* Ask a question on the [#kured](https://weave-community.slack.com/messages/kured/) slack channel.
-* [File an issue](https://github.com/weaveworks/kured/issues/new).
-* Join us in [our monthly meeting](https://docs.google.com/document/d/1bsHTjHhqaaZ7yJnXF6W8c89UB_yn-OoSZEmDnIP34n8/edit#),
-  every fourth Wednesday of the month at 16:00 UTC.
+- Invite yourself to the <a href="https://slack.cncf.io/" target="_blank">CNCF Slack</a>.
+- Ask a question on the [#kured](https://cloud-native.slack.com/archives/kured) slack channel.
+- [File an issue](https://github.com/kubereboot/kured/issues/new).
+- Join us in [our monthly meeting](https://docs.google.com/document/d/1AWT8YDdqZY-Se6Y1oAlwtujWLVpNVK2M_F_Vfqw06aI/edit),
+  every first Wednesday of the month at 16:00 UTC.
+- You might want to [join the kured-dev mailing list](https://lists.cncf.io/g/cncf-kured-dev) as well.
+
+We follow the [CNCF Code of Conduct](CODE_OF_CONDUCT.md).
 
 Your feedback is always welcome!
+
+## Trademarks
+
+**Kured is a [Cloud Native Computing Foundation](https://cncf.io/) Sandbox project.**
+
+![Cloud Native Computing Foundation logo](img/cncf-color.png)
+
+The Linux Foundation® (TLF) has registered trademarks and uses trademarks. For a list of TLF trademarks, see [Trademark Usage](https://www.linuxfoundation.org/trademark-usage/).
+
+## License
+
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkubereboot%2Fkured.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkubereboot%2Fkured?ref=badge_large)

charts/kured/.helmignore

@@ -1,21 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj

charts/kured/Chart.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-appVersion: "1.5.1"
-description: A Helm chart for kured
-name: kured
-version: 2.2.3
-home: https://github.com/weaveworks/kured
-maintainers:
-  - name: ckotzbauer
-    email: christian.kotzbauer@gmail.com
-  - name: davidkarlsen
-    email: david@davidkarlsen.com
-sources:
-  - https://github.com/weaveworks/kured
-icon: https://raw.githubusercontent.com/weaveworks/kured/master/img/logo.png

charts/kured/README.md

@@ -1,108 +0,0 @@
-# Kured (KUbernetes REboot Daemon)
-
-## Introduction
-This chart installs the "Kubernetes Reboot Daemon" using the Helm Package Manager.
-
-## Prerequisites
-- Kubernetes 1.9+
-
-## Installing the Chart
-To install the chart with the release name `my-release`:
-```bash
-$ helm repo add kured https://weaveworks.github.io/kured
-$ helm install my-release kured/kured
-```
-
-## Uninstalling the Chart
-To uninstall/delete the `my-release` deployment:
-```bash
-$ helm delete my-release
-```
-
-The command removes all the Kubernetes components associated with the chart and deletes the release.
-
-
-## Migrate from stable Helm-Chart
-The following changes have been made compared to the stable chart:
-- **[BREAKING CHANGE]** The `autolock` feature was removed. Use `configuration.startTime` and `configuration.endTime` instead.
-- Role inconsistencies have been fixed (allowed verbs for modifying the `DaemonSet`, apiGroup of `PodSecurityPolicy`)
-- Added support for affinities.
-- Configuration of cli-flags can be made through a `configuration` object.
-- Added optional `Service` and `ServiceMonitor` support for metrics endpoint.
-
-
-## Configuration
-
-| Config                  | Description                                                                 | Default                    |
-| ------                  | -----------                                                                 | -------                    |
-| `image.repository`      | Image repository                                                            | `weaveworks/kured` |
-| `image.tag`             | Image tag                                                                   | `1.5.1`                    |
-| `image.pullPolicy`      | Image pull policy                                                           | `IfNotPresent`             |
-| `image.pullSecrets`     | Image pull secrets                                                          | `[]`                       |
-| `updateStrategy`        | Daemonset update strategy                                                   | `OnDelete`                 |
-| `podAnnotations`        | Annotations to apply to pods (eg to add Prometheus annotations)             | `{}`                       |
-| `extraArgs`             | Extra arguments to pass to `/usr/bin/kured`. See below.                     | `{}`                       |
-| `extraEnvVars`          | Array of environment variables to pass to the daemonset.                    | `{}`                       |
-| `configuration.lockTtl` | cli-parameter `--lock-ttl`                                                  | `0`                       |
-| `configuration.alertFilterRegexp` | cli-parameter `--alert-filter-regexp`                             | `""`                       |
-| `configuration.blockingPodSelector` | Array of selectors for multiple cli-parameters `--blocking-pod-selector` | `[]`             |
-| `configuration.endTime` | cli-parameter `--end-time`                                                  | `""`                      |
-| `configuration.lockAnnotation` | cli-parameter `--lock-annotation`                                    | `""`                      |
-| `configuration.period` | cli-parameter `--period`                                                     | `""`                      |
-| `configuration.prometheusUrl` | cli-parameter `--prometheus-url`                                      | `""`                      |
-| `configuration.rebootDays` | Array of days for multiple cli-parameters `--reboot-days`                | `[]`                      |
-| `configuration.rebootSentinel` | cli-parameter `--reboot-sentinel`                                    | `""`                      |
-| `configuration.slackChannel` | cli-parameter `--slack-channel`                                        | `""`                      |
-| `configuration.slackHookUrl` | cli-parameter `--slack-hook-url`                                       | `""`                      |
-| `configuration.slackUsername` | cli-parameter `--slack-username`                                      | `""`                      |
-| `configuration.messageTemplateDrain` | cli-parameter `--message-template-drain`                       | `""`                      |
-| `configuration.messageTemplateReboot` | cli-parameter `--message-template-reboot`                     | `""`                      |
-| `configuration.startTime` | cli-parameter `--start-time`                                              | `""`                      |
-| `configuration.timeZone` | cli-parameter `--time-zone`                                                | `""`                      |
-| `rbac.create`           | Create RBAC roles                                                           | `true`                     |
-| `serviceAccount.create` | Create a service account                                                    | `true`                     |
-| `serviceAccount.name`   | Service account name to create (or use if `serviceAccount.create` is false) | (chart fullname)           |
-| `podSecurityPolicy.create` | Create podSecurityPolicy                                                 | `false`                     |
-| `resources`             | Resources requests and limits.                                              | `{}`                       |
-| `metrics.create`        | Create a ServiceMonitor for prometheus-operator                             | `false`                    |
-| `metrics.namespace`     | The namespace to create the ServiceMonitor in                               | `""`                    |
-| `metrics.labels`        | Additional labels for the ServiceMonitor                                    | `{}`                    |
-| `metrics.interval`      | Interval prometheus should scrape the endpoint                              | `60s`                   |
-| `metrics.scrapeTimeout` | A custom scrapeTimeout for prometheus                                       | `""`                    |
-| `service.create`        | Create a Service for the metrics endpoint                                   | `false`                    |
-| `service.port`          | Port of the service to expose                                               | `8080`                     |
-| `service.annotations`   | Annotations to apply to the service (eg to add Prometheus annotations)      | `{}`                       |
-| `priorityClassName`     | Priority Class to be used by the pods                                       | `""`                       |
-| `tolerations`           | Tolerations to apply to the daemonset (eg to allow running on master)       | `[{"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}]`|
-| `affinity`              | Affinity for the daemonset (ie, restrict which nodes kured runs on)         | `{}`                       |
-| `nodeSelector`          | Node Selector for the daemonset (ie, restrict which nodes kured runs on)    | `{}`                       |
-
-See https://github.com/weaveworks/kured#configuration for values (not contained in the `configuration` object) for `extraArgs`. Note that
-```yaml
-extraArgs:
-  foo: 1
-  bar-baz: 2
-```
-becomes `/usr/bin/kured ... --foo=1 --bar-baz=2`.
-
-
-## Prometheus Metrics
-
-Kured exposes a single prometheus metric indicating whether a reboot is required or not (see [kured docs](https://github.com/weaveworks/kured#prometheus-metrics)) for details.
-
-#### Prometheus-Operator
-
-```yaml
-metrics:
-  create: true
-```
-
-#### Prometheus Annotations
-
-```yaml
-service:
-  annotations:
-    prometheus.io/scrape: "true"
-    prometheus.io/path: "/metrics"
-    prometheus.io/port: "8080"
-```

charts/kured/templates/NOTES.txt

@@ -1,3 +0,0 @@
-Kured will check for /var/run/reboot-required, and reboot nodes when needed.
-
-See https://github.com/weaveworks/kured/ for details.

charts/kured/templates/_helpers.tpl

@@ -1,72 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "kured.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "kured.fullname" -}}
-{{- if .Values.fullnameOverride -}}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- if contains $name .Release.Name -}}
-{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "kured.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "kured.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create -}}
-    {{ default (include "kured.fullname" .) .Values.serviceAccount.name }}
-{{- else -}}
-    {{ default "default" .Values.serviceAccount.name }}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Return the appropriate apiVersion for podsecuritypolicy.
-*/}}
-{{- define "kured.psp.apiVersion" -}}
-{{- if semverCompare "<1.10-0" .Capabilities.KubeVersion.GitVersion -}}
-{{- print "extensions/v1beta1" -}}
-{{- else -}}
-{{- print "policy/v1beta1" -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Returns a set of labels applied to each resource.
-*/}}
-{{- define "kured.labels" -}}
-app: {{ template "kured.name" . }}
-chart: {{ template "kured.chart" . }}
-release: {{ .Release.Name }}
-heritage: {{ .Release.Service }}
-{{- end -}}
-
-{{/*
-Returns a set of matchLabels applied.
-*/}}
-{{- define "kured.matchLabels" -}}
-app: {{ template "kured.name" . }}
-release: {{ .Release.Name }}
-{{- end -}}

charts/kured/templates/clusterrole.yaml

@@ -1,30 +0,0 @@
-{{- if .Values.rbac.create -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: {{ template "kured.fullname" . }}
-  labels:
-    {{- include "kured.labels" . | nindent 4 }}
-rules:
-# Allow kured to read spec.unschedulable
-# Allow kubectl to drain/uncordon
-#
-# NB: These permissions are tightly coupled to the bundled version of kubectl; the ones below
-# match https://github.com/kubernetes/kubernetes/blob/v1.19.4/staging/src/k8s.io/kubectl/pkg/cmd/drain/drain.go
-#
-- apiGroups: [""]
-  resources: ["nodes"]
-  verbs:     ["get", "patch"]
-- apiGroups: [""]
-  resources: ["pods"]
-  verbs:     ["list","delete","get"]
-- apiGroups: ["extensions"]
-  resources: ["daemonsets"]
-  verbs:     ["get"]
-- apiGroups: ["apps"]
-  resources: ["daemonsets"]
-  verbs:     ["get"]
-- apiGroups: [""]
-  resources: ["pods/eviction"]
-  verbs:     ["create"]
-{{- end -}}

charts/kured/templates/clusterrolebinding.yaml

@@ -1,16 +0,0 @@
-{{- if .Values.rbac.create -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: {{ template "kured.fullname" . }}
-  labels:
-    {{- include "kured.labels" . | nindent 4 }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ template "kured.fullname" . }}
-subjects:
-- kind: ServiceAccount
-  name: {{ template "kured.serviceAccountName" . }}
-  namespace: {{ .Release.Namespace }}
-{{- end -}}

charts/kured/templates/daemonset.yaml

@@ -1,127 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: {{ template "kured.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "kured.labels" . | nindent 4 }}
-spec:
-  updateStrategy:
-    type: {{ .Values.updateStrategy }}
-  selector:
-    matchLabels:
-      {{- include "kured.matchLabels" . | nindent 6 }}
-  template:
-    metadata:
-      labels:
-        {{- include "kured.labels" . | nindent 8 }}
-      {{- if .Values.podAnnotations }}
-      annotations:
-      {{- range $key, $value := .Values.podAnnotations }}
-        {{ $key }}: {{ $value | quote }}
-      {{- end }}
-      {{- end }}
-    spec:
-      serviceAccountName: {{ template "kured.serviceAccountName" . }}
-      hostPID: true
-      restartPolicy: Always
-      {{- with .Values.image.pullSecrets }}
-      imagePullSecrets:
-{{ toYaml . | indent 8 }}
-      {{- end }}
-      {{- if .Values.priorityClassName }}
-      priorityClassName: {{ .Values.priorityClassName }}
-      {{- end }}
-      containers:
-        - name: {{ .Chart.Name }}
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          securityContext:
-            privileged: true # Give permission to nsenter /proc/1/ns/mnt
-          resources:
-{{ toYaml .Values.resources | indent 12 }}
-          command:
-            - /usr/bin/kured
-          args:
-            - --ds-name={{ template "kured.fullname" . }}
-            - --ds-namespace={{ .Release.Namespace }}
-          {{- if .Values.configuration.lockTtl }}
-            - --lock-ttl={{ .Values.configuration.lockTtl }}
-          {{- end }}
-          {{- if .Values.configuration.alertFilterRegexp }}
-            - --alert-filter-regexp={{ .Values.configuration.alertFilterRegexp }}
-          {{- end }}
-          {{- range .Values.configuration.blockingPodSelector }}
-            - --blocking-pod-selector={{ . }}
-          {{- end }}
-          {{- if .Values.configuration.endTime }}
-            - --end-time={{ .Values.configuration.endTime }}
-          {{- end }}
-          {{- if .Values.configuration.lockAnnotation }}
-            - --lock-annotation={{ .Values.configuration.lockAnnotation }}
-          {{- end }}
-          {{- if .Values.configuration.period }}
-            - --period={{ .Values.configuration.period }}
-          {{- end }}
-          {{- if .Values.configuration.prometheusUrl }}
-            - --prometheus-url={{ .Values.configuration.prometheusUrl }}
-          {{- end }}
-          {{- range .Values.configuration.rebootDays }}
-            - --reboot-days={{ . }}
-          {{- end }}
-          {{- if .Values.configuration.rebootSentinel }}
-            - --reboot-sentinel={{ .Values.configuration.rebootSentinel }}
-          {{- end }}
-          {{- if .Values.configuration.slackChannel }}
-            - --slack-channel={{ .Values.configuration.slackChannel }}
-          {{- end }}
-          {{- if .Values.configuration.slackHookUrl }}
-            - --slack-hook-url={{ .Values.configuration.slackHookUrl }}
-          {{- end }}
-          {{- if .Values.configuration.slackUsername }}
-            - --slack-username={{ .Values.configuration.slackUsername }}
-          {{- end }}
-          {{- if .Values.configuration.messageTemplateDrain }}
-            - --message-template-drain={{ .Values.configuration.messageTemplateDrain }}
-          {{- end }}
-          {{- if .Values.configuration.messageTemplateReboot }}
-            - --message-template-reboot={{ .Values.configuration.messageTemplateReboot }}
-          {{- end }}
-          {{- if .Values.configuration.startTime }}
-            - --start-time={{ .Values.configuration.startTime }}
-          {{- end }}
-          {{- if .Values.configuration.timeZone }}
-            - --time-zone={{ .Values.configuration.timeZone }}
-          {{- end }}
-          {{- range $key, $value := .Values.extraArgs }}
-            {{- if $value }}
-            - --{{ $key }}={{ $value }}
-            {{- else }}
-            - --{{ $key }}
-            {{- end }}
-          {{- end }}
-          ports:
-            - containerPort: 8080
-              name: metrics
-          env:
-            # Pass in the name of the node on which this pod is scheduled
-            # for use with drain/uncordon operations and lock acquisition
-            - name: KURED_NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            {{- if .Values.extraEnvVars }}
-              {{ toYaml .Values.extraEnvVars | nindent 12 }}
-            {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-{{ toYaml . | indent 8 }}
-      {{- end }}
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-{{ toYaml . | indent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-{{ toYaml . | indent 8 }}
-      {{- end }}

charts/kured/templates/podsecuritypolicy.yaml

@@ -1,21 +0,0 @@
-{{- if .Values.podSecurityPolicy.create}}
-apiVersion: {{ template "kured.psp.apiVersion" . }}
-kind: PodSecurityPolicy
-metadata:
-  name: {{ template "kured.fullname" . }}
-  labels:
-    {{- include "kured.labels" . | nindent 4 }}
-spec:
-  privileged: true
-  hostPID: true
-  allowedCapabilities: ['*']
-  fsGroup:
-    rule: RunAsAny
-  runAsUser:
-    rule: RunAsAny
-  seLinux:
-    rule: RunAsAny
-  supplementalGroups:
-    rule: RunAsAny
-  volumes: ['*']
-{{- end }}

charts/kured/templates/role.yaml

@@ -1,30 +0,0 @@
-{{- if .Values.rbac.create -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  namespace: {{ .Release.Namespace }}
-  name: {{ template "kured.fullname" . }}
-  labels:
-    {{- include "kured.labels" . | nindent 4 }}
-rules:
-  # Allow kured to lock/unlock itself
-  - apiGroups:     ["extensions"]
-    resources:     ["daemonsets"]
-    resourceNames: ["{{ template "kured.fullname" . }}"]
-    verbs:         ["update", "patch"]
-  - apiGroups:     ["apps"]
-    resources:     ["daemonsets"]
-    resourceNames: ["{{ template "kured.fullname" . }}"]
-    verbs:         ["update", "patch"]
-{{- if .Values.podSecurityPolicy.create }}
-  - apiGroups:     ["extensions"]
-    resources:     ["podsecuritypolicies"]
-    resourceNames: ["{{ template "kured.fullname" . }}"]
-    verbs:         ["use"]
-  - apiGroups:     ["policy"]
-    resources:     ["podsecuritypolicies"]
-    resourceNames: ["{{ template "kured.fullname" . }}"]
-    verbs:         ["use"]
-{{- end }}
-
-{{- end -}}

charts/kured/templates/rolebinding.yaml

@@ -1,17 +0,0 @@
-{{- if .Values.rbac.create -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  namespace: {{ .Release.Namespace }}
-  name: {{ template "kured.fullname" . }}
-  labels:
-    {{- include "kured.labels" . | nindent 4 }}
-subjects:
-- kind: ServiceAccount
-  namespace: {{ .Release.Namespace }}
-  name: {{ template "kured.serviceAccountName" . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: {{ template "kured.fullname" . }}
-{{- end -}}

charts/kured/templates/service.yaml

@@ -1,22 +0,0 @@
-{{- if or .Values.service.create .Values.metrics.create }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ template "kured.fullname" . }}
-  labels:
-    {{- include "kured.labels" . | nindent 4 }}
-  {{- if .Values.service.annotations }}
-  annotations:
-  {{- range $key, $value := .Values.service.annotations }}
-    {{ $key }}: {{ $value | quote }}
-  {{- end }}
-  {{- end }}
-spec:
-  type: ClusterIP
-  ports:
-    - name: metrics
-      port: {{ .Values.service.port }}
-      targetPort: 8080
-  selector:
-    {{- include "kured.matchLabels" . | nindent 4 }}
-{{- end }}

charts/kured/templates/serviceaccount.yaml

@@ -1,9 +0,0 @@
-{{- if .Values.serviceAccount.create -}}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ template "kured.serviceAccountName" . }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "kured.labels" . | nindent 4 }}
-{{- end -}}

charts/kured/templates/servicemonitor.yaml

@@ -1,31 +0,0 @@
-{{- if .Values.metrics.create }}
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: {{ template "kured.fullname" . }}
-  {{- if .Values.metrics.namespace }}
-  namespace: {{ .Values.metrics.namespace }}
-  {{- end }}
-  labels:
-    {{- include "kured.labels" . | nindent 4 }}
-    {{- if .Values.metrics.labels }}
-    {{- toYaml .Values.metrics.labels | nindent 4 }}
-    {{- end }}
-spec:
-  endpoints:
-  - interval: {{ .Values.metrics.interval }}
-    {{- if .Values.metrics.scrapeTimeout }}
-    scrapeTimeout: {{ .Values.metrics.scrapeTimeout }}
-    {{- end }}
-    honorLabels: true
-    targetPort: 8080
-    path: /metrics
-    scheme: http
-  jobLabel: "{{ .Release.Name }}"
-  selector:
-    matchLabels:
-      {{- include "kured.matchLabels" . | nindent 6 }}
-  namespaceSelector:
-    matchNames:
-      - {{ .Release.Namespace }}
-{{- end }}

charts/kured/values.minikube.yaml

@@ -1,21 +0,0 @@
-image:
-  repository: weaveworks/kured
-  tag: latest
-
-configuration:
-  # annotationTtl: 0          # force clean annotation after this ammount of time (default 0, disabled)
-  # alertFilterRegexp: ""     # alert names to ignore when checking for active alerts
-  # blockingPodSelector: []   # label selector identifying pods whose presence should prevent reboots
-  # endTime: ""               # only reboot before this time of day (default "23:59")
-  # lockAnnotation: ""        # annotation in which to record locking node (default "weave.works/kured-node-lock")
-  period: "1m"                # reboot check period (default 1h0m0s)
-  # prometheusUrl: ""         # Prometheus instance to probe for active alerts
-  # rebootDays: []            # only reboot on these days (default [su,mo,tu,we,th,fr,sa])
-  # rebootSentinel: ""        # path to file whose existence signals need to reboot (default "/var/run/reboot-required")
-  # slackChannel: ""          # slack channel for reboot notfications
-  # slackHookUrl: ""          # slack hook URL for reboot notfications
-  # slackUsername: ""         # slack username for reboot notfications (default "kured")
-  # messageTemplateDrain: ""  # slack message template when notifying about a node being drained (default "Draining node %s")
-  # messageTemplateReboot: "" # slack message template when notifying about a node being rebooted (default "Rebooted node %s")
-  # startTime: ""             # only reboot after this time of day (default "0:00")
-  # timeZone: ""              # time-zone to use (valid zones from "time" golang package)

charts/kured/values.yaml

@@ -1,72 +0,0 @@
-image:
-  repository: weaveworks/kured
-  tag: 1.5.1
-  pullPolicy: IfNotPresent
-  pullSecrets: []
-
-updateStrategy: OnDelete
-
-podAnnotations: {}
-
-extraArgs: {}
-
-extraEnvVars:
-#  - name: slackHookUrl
-#    valueFrom:
-#      secretKeyRef:
-#        name: secret_name
-#        key: secret_key
-#  - name: regularEnvVariable
-#    value: 123
-
-configuration:
-  lockTtl: 0                 # force clean annotation after this ammount of time (default 0, disabled)
-  alertFilterRegexp: ""      # alert names to ignore when checking for active alerts
-  blockingPodSelector: []    # label selector identifying pods whose presence should prevent reboots
-  endTime: ""                # only reboot before this time of day (default "23:59")
-  lockAnnotation: ""         # annotation in which to record locking node (default "weave.works/kured-node-lock")
-  period: ""                 # reboot check period (default 1h0m0s)
-  prometheusUrl: ""          # Prometheus instance to probe for active alerts
-  rebootDays: []             # only reboot on these days (default [su,mo,tu,we,th,fr,sa])
-  rebootSentinel: ""         # path to file whose existence signals need to reboot (default "/var/run/reboot-required")
-  slackChannel: ""           # slack channel for reboot notfications
-  slackHookUrl: ""           # slack hook URL for reboot notfications
-  slackUsername: ""          # slack username for reboot notfications (default "kured")
-  messageTemplateDrain: ""   # slack message template when notifying about a node being drained (default "Draining node %s")
-  messageTemplateReboot: ""  # slack message template when notifying about a node being rebooted (default "Rebooted node %s")
-  startTime: ""              # only reboot after this time of day (default "0:00")
-  timeZone: ""               # time-zone to use (valid zones from "time" golang package)
-
-rbac:
-  create: true
-
-serviceAccount:
-  create: true
-  name:
-
-podSecurityPolicy:
-  create: false
-
-resources: {}
-
-metrics:
-  create: false
-  namespace: ""
-  labels: {}
-  interval: 60s
-  scrapeTimeout: ""
-
-service:
-  create: false
-  port: 8080
-  annotations: {}
-
-priorityClassName: ""
-
-tolerations:
-  - key: node-role.kubernetes.io/master
-    effect: NoSchedule
-
-affinity: {}
-
-nodeSelector: {}

cmd/kured/Dockerfile

@@ -1,4 +0,0 @@
-FROM alpine:3.12
-RUN apk update --no-cache && apk upgrade --no-cache && apk add --no-cache ca-certificates tzdata
-COPY ./kured /usr/bin/kured
-ENTRYPOINT ["/usr/bin/kured"]

cmd/kured/main.go

@@ -2,54 +2,93 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
+	"reflect"
 	"regexp"
+	"sort"
+	"strings"
 	"time"
 
+	papi "github.com/prometheus/client_golang/api"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	kubectldrain "k8s.io/kubectl/pkg/drain"
 
+	"github.com/google/shlex"
+
+	shoutrrr "github.com/containrrr/shoutrrr"
+	"github.com/kubereboot/kured/pkg/alerts"
+	"github.com/kubereboot/kured/pkg/daemonsetlock"
+	"github.com/kubereboot/kured/pkg/delaytick"
+	"github.com/kubereboot/kured/pkg/reboot"
+	"github.com/kubereboot/kured/pkg/taints"
+	"github.com/kubereboot/kured/pkg/timewindow"
+	"github.com/kubereboot/kured/pkg/util"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/weaveworks/kured/pkg/alerts"
-	"github.com/weaveworks/kured/pkg/daemonsetlock"
-	"github.com/weaveworks/kured/pkg/delaytick"
-	"github.com/weaveworks/kured/pkg/notifications/slack"
-	"github.com/weaveworks/kured/pkg/timewindow"
 )
 
 var (
 	version = "unreleased"
 
 	// Command line flags
-	period                 time.Duration
-	dsNamespace            string
-	dsName                 string
-	lockAnnotation         string
-	lockTTL                time.Duration
-	prometheusURL          string
-	alertFilter            *regexp.Regexp
-	rebootSentinel         string
-	slackHookURL           string
-	slackUsername          string
-	slackChannel           string
-	messageTemplateDrain   string
-	messageTemplateReboot  string
-	podSelectors           []string
+	forceReboot                     bool
+	drainDelay                      time.Duration
+	drainTimeout                    time.Duration
+	rebootDelay                     time.Duration
+	rebootMethod                    string
+	period                          time.Duration
+	metricsHost                     string
+	metricsPort                     int
+	drainGracePeriod                int
+	drainPodSelector                string
+	skipWaitForDeleteTimeoutSeconds int
+	dsNamespace                     string
+	dsName                          string
+	lockAnnotation                  string
+	lockTTL                         time.Duration
+	lockReleaseDelay                time.Duration
+	prometheusURL                   string
+	preferNoScheduleTaintName       string
+	alertFilter                     *regexp.Regexp
+	alertFilterMatchOnly            bool
+	alertFiringOnly                 bool
+	rebootSentinelFile              string
+	rebootSentinelCommand           string
+	notifyURL                       string
+	slackHookURL                    string
+	slackUsername                   string
+	slackChannel                    string
+	messageTemplateDrain            string
+	messageTemplateReboot           string
+	messageTemplateUncordon         string
+	podSelectors                    []string
+	rebootCommand                   string
+	rebootSignal                    int
+	logFormat                       string
+	preRebootNodeLabels             []string
+	postRebootNodeLabels            []string
+	nodeID                          string
+	concurrency                     int
 
-	rebootDays  []string
-	rebootStart string
-	rebootEnd   string
-	timezone    string
+	rebootDays    []string
+	rebootStart   string
+	rebootEnd     string
+	timezone      string
+	annotateNodes bool
 
 	// Metrics
 	rebootRequiredGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -59,39 +98,110 @@ var (
 	}, []string{"node"})
 )
 
+const (
+	// KuredNodeLockAnnotation is the canonical string value for the kured node-lock annotation
+	KuredNodeLockAnnotation string = "weave.works/kured-node-lock"
+	// KuredRebootInProgressAnnotation is the canonical string value for the kured reboot-in-progress annotation
+	KuredRebootInProgressAnnotation string = "weave.works/kured-reboot-in-progress"
+	// KuredMostRecentRebootNeededAnnotation is the canonical string value for the kured most-recent-reboot-needed annotation
+	KuredMostRecentRebootNeededAnnotation string = "weave.works/kured-most-recent-reboot-needed"
+	// EnvPrefix The environment variable prefix of all environment variables bound to our command line flags.
+	EnvPrefix = "KURED"
+
+	// MethodCommand is used as "--reboot-method" value when rebooting with the configured "--reboot-command"
+	MethodCommand = "command"
+	// MethodSignal is used as "--reboot-method" value when rebooting with a SIGRTMIN+5 signal.
+	MethodSignal = "signal"
+
+	sigTrminPlus5 = 34 + 5
+)
+
 func init() {
 	prometheus.MustRegister(rebootRequiredGauge)
 }
 
 func main() {
-	rootCmd := &cobra.Command{
-		Use:   "kured",
-		Short: "Kubernetes Reboot Daemon",
-		Run:   root}
+	cmd := NewRootCommand()
 
+	if err := cmd.Execute(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// NewRootCommand construct the Cobra root command
+func NewRootCommand() *cobra.Command {
+	rootCmd := &cobra.Command{
+		Use:               "kured",
+		Short:             "Kubernetes Reboot Daemon",
+		PersistentPreRunE: bindViper,
+		PreRun:            flagCheck,
+		Run:               root}
+
+	rootCmd.PersistentFlags().StringVar(&nodeID, "node-id", "",
+		"node name kured runs on, should be passed down from spec.nodeName via KURED_NODE_ID environment variable")
+	rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false,
+		"force a reboot even if the drain fails or times out")
+	rootCmd.PersistentFlags().StringVar(&metricsHost, "metrics-host", "",
+		"host where metrics will listen")
+	rootCmd.PersistentFlags().IntVar(&metricsPort, "metrics-port", 8080,
+		"port number where metrics will listen")
+	rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1,
+		"time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used")
+	rootCmd.PersistentFlags().StringVar(&drainPodSelector, "drain-pod-selector", "",
+		"only drain pods with labels matching the selector (default: '', all pods)")
+	rootCmd.PersistentFlags().IntVar(&skipWaitForDeleteTimeoutSeconds, "skip-wait-for-delete-timeout", 0,
+		"when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node")
+	rootCmd.PersistentFlags().DurationVar(&drainDelay, "drain-delay", 0,
+		"delay drain for this duration (default: 0, disabled)")
+	rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 0,
+		"timeout after which the drain is aborted (default: 0, infinite time)")
+	rootCmd.PersistentFlags().DurationVar(&rebootDelay, "reboot-delay", 0,
+		"delay reboot for this duration (default: 0, disabled)")
+	rootCmd.PersistentFlags().StringVar(&rebootMethod, "reboot-method", "command",
+		"method to use for reboots. Available: command")
 	rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60,
-		"reboot check period")
+		"sentinel check period")
 	rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system",
 		"namespace containing daemonset on which to place lock")
 	rootCmd.PersistentFlags().StringVar(&dsName, "ds-name", "kured",
 		"name of daemonset on which to place lock")
-	rootCmd.PersistentFlags().StringVar(&lockAnnotation, "lock-annotation", "weave.works/kured-node-lock",
+	rootCmd.PersistentFlags().StringVar(&lockAnnotation, "lock-annotation", KuredNodeLockAnnotation,
 		"annotation in which to record locking node")
 	rootCmd.PersistentFlags().DurationVar(&lockTTL, "lock-ttl", 0,
 		"expire lock annotation after this duration (default: 0, disabled)")
+	rootCmd.PersistentFlags().DurationVar(&lockReleaseDelay, "lock-release-delay", 0,
+		"delay lock release for this duration (default: 0, disabled)")
 	rootCmd.PersistentFlags().StringVar(&prometheusURL, "prometheus-url", "",
 		"Prometheus instance to probe for active alerts")
 	rootCmd.PersistentFlags().Var(&regexpValue{&alertFilter}, "alert-filter-regexp",
 		"alert names to ignore when checking for active alerts")
-	rootCmd.PersistentFlags().StringVar(&rebootSentinel, "reboot-sentinel", "/var/run/reboot-required",
-		"path to file whose existence signals need to reboot")
+	rootCmd.PersistentFlags().BoolVar(&alertFilterMatchOnly, "alert-filter-match-only", false,
+		"Only block if the alert-filter-regexp matches active alerts")
+	rootCmd.PersistentFlags().BoolVar(&alertFiringOnly, "alert-firing-only", false,
+		"only consider firing alerts when checking for active alerts")
+	rootCmd.PersistentFlags().StringVar(&rebootSentinelFile, "reboot-sentinel", "/var/run/reboot-required",
+		"path to file whose existence triggers the reboot command")
+	rootCmd.PersistentFlags().StringVar(&preferNoScheduleTaintName, "prefer-no-schedule-taint", "",
+		"Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Disabled by default. Set e.g. to \"weave.works/kured-node-reboot\" to enable tainting.")
+	rootCmd.PersistentFlags().StringVar(&rebootSentinelCommand, "reboot-sentinel-command", "",
+		"command for which a zero return code will trigger a reboot command")
+	rootCmd.PersistentFlags().StringVar(&rebootCommand, "reboot-command", "/bin/systemctl reboot",
+		"command to run when a reboot is required")
+	rootCmd.PersistentFlags().IntVar(&concurrency, "concurrency", 1,
+		"amount of nodes to concurrently reboot. Defaults to 1")
+	rootCmd.PersistentFlags().IntVar(&rebootSignal, "reboot-signal", sigTrminPlus5,
+		"signal to use for reboot, SIGRTMIN+5 by default.")
 
 	rootCmd.PersistentFlags().StringVar(&slackHookURL, "slack-hook-url", "",
-		"slack hook URL for reboot notfications")
+		"slack hook URL for reboot notifications [deprecated in favor of --notify-url]")
 	rootCmd.PersistentFlags().StringVar(&slackUsername, "slack-username", "kured",
-		"slack username for reboot notfications")
+		"slack username for reboot notifications")
 	rootCmd.PersistentFlags().StringVar(&slackChannel, "slack-channel", "",
-		"slack channel for reboot notfications")
+		"slack channel for reboot notifications")
+	rootCmd.PersistentFlags().StringVar(&notifyURL, "notify-url", "",
+		"notify URL for reboot notifications (cannot use with --slack-hook-url flags)")
+	rootCmd.PersistentFlags().StringVar(&messageTemplateUncordon, "message-template-uncordon", "Node %s rebooted & uncordoned successfully!",
+		"message template used to notify about a node being successfully uncordoned")
 	rootCmd.PersistentFlags().StringVar(&messageTemplateDrain, "message-template-drain", "Draining node %s",
 		"message template used to notify about a node being drained")
 	rootCmd.PersistentFlags().StringVar(&messageTemplateReboot, "message-template-reboot", "Rebooting node %s",
@@ -109,32 +219,115 @@ func main() {
 	rootCmd.PersistentFlags().StringVar(&timezone, "time-zone", "UTC",
 		"use this timezone for schedule inputs")
 
-	if err := rootCmd.Execute(); err != nil {
-		log.Fatal(err)
+	rootCmd.PersistentFlags().BoolVar(&annotateNodes, "annotate-nodes", false,
+		"if set, the annotations 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' will be given to nodes undergoing kured reboots")
+
+	rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "text",
+		"use text or json log format")
+
+	rootCmd.PersistentFlags().StringSliceVar(&preRebootNodeLabels, "pre-reboot-node-labels", nil,
+		"labels to add to nodes before cordoning")
+	rootCmd.PersistentFlags().StringSliceVar(&postRebootNodeLabels, "post-reboot-node-labels", nil,
+		"labels to add to nodes after uncordoning")
+
+	return rootCmd
+}
+
+// func that checks for deprecated slack-notification-related flags and node labels that do not match
+func flagCheck(cmd *cobra.Command, args []string) {
+	if slackHookURL != "" && notifyURL != "" {
+		log.Warnf("Cannot use both --notify-url and --slack-hook-url flags. Kured will use --notify-url flag only...")
+	}
+	if notifyURL != "" {
+		notifyURL = stripQuotes(notifyURL)
+	} else if slackHookURL != "" {
+		slackHookURL = stripQuotes(slackHookURL)
+		log.Warnf("Deprecated flag(s). Please use --notify-url flag instead.")
+		trataURL, err := url.Parse(slackHookURL)
+		if err != nil {
+			log.Warnf("slack-hook-url is not properly formatted... no notification will be sent: %v\n", err)
+		}
+		if len(strings.Split(strings.Trim(trataURL.Path, "/services/"), "/")) != 3 {
+			log.Warnf("slack-hook-url is not properly formatted... no notification will be sent: unexpected number of / in URL\n")
+		} else {
+			notifyURL = fmt.Sprintf("slack://%s", strings.Trim(trataURL.Path, "/services/"))
+		}
+	}
+	var preRebootNodeLabelKeys, postRebootNodeLabelKeys []string
+	for _, label := range preRebootNodeLabels {
+		preRebootNodeLabelKeys = append(preRebootNodeLabelKeys, strings.Split(label, "=")[0])
+	}
+	for _, label := range postRebootNodeLabels {
+		postRebootNodeLabelKeys = append(postRebootNodeLabelKeys, strings.Split(label, "=")[0])
+	}
+	sort.Strings(preRebootNodeLabelKeys)
+	sort.Strings(postRebootNodeLabelKeys)
+	if !reflect.DeepEqual(preRebootNodeLabelKeys, postRebootNodeLabelKeys) {
+		log.Warnf("pre-reboot-node-labels keys and post-reboot-node-labels keys do not match. This may result in unexpected behaviour.")
 	}
 }
 
-// newCommand creates a new Command with stdout/stderr wired to our standard logger
-func newCommand(name string, arg ...string) *exec.Cmd {
-	cmd := exec.Command(name, arg...)
+// stripQuotes removes any literal single or double quote chars that surround a string
+func stripQuotes(str string) string {
+	if len(str) > 2 {
+		firstChar := str[0]
+		lastChar := str[len(str)-1]
+		if firstChar == lastChar && (firstChar == '"' || firstChar == '\'') {
+			return str[1 : len(str)-1]
+		}
+	}
+	// return the original string if it has a length of zero or one
+	return str
+}
 
-	cmd.Stdout = log.NewEntry(log.StandardLogger()).
-		WithField("cmd", cmd.Args[0]).
-		WithField("std", "out").
-		WriterLevel(log.InfoLevel)
+// bindViper initializes viper and binds command flags with environment variables
+func bindViper(cmd *cobra.Command, args []string) error {
+	v := viper.New()
 
-	cmd.Stderr = log.NewEntry(log.StandardLogger()).
-		WithField("cmd", cmd.Args[0]).
-		WithField("std", "err").
-		WriterLevel(log.WarnLevel)
+	v.SetEnvPrefix(EnvPrefix)
+	v.AutomaticEnv()
+	bindFlags(cmd, v)
 
+	return nil
+}
+
+// bindFlags binds each cobra flag to its associated viper configuration (environment variable)
+func bindFlags(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().VisitAll(func(f *pflag.Flag) {
+		// Environment variables can't have dashes in them, so bind them to their equivalent keys with underscores
+		if strings.Contains(f.Name, "-") {
+			v.BindEnv(f.Name, flagToEnvVar(f.Name))
+		}
+
+		// Apply the viper config value to the flag when the flag is not set and viper has a value
+		if !f.Changed && v.IsSet(f.Name) {
+			val := v.Get(f.Name)
+			log.Infof("Binding %s command flag to environment variable: %s", f.Name, flagToEnvVar(f.Name))
+			cmd.Flags().Set(f.Name, fmt.Sprintf("%v", val))
+		}
+	})
+}
+
+// flagToEnvVar converts command flag name to equivalent environment variable name
+func flagToEnvVar(flag string) string {
+	envVarSuffix := strings.ToUpper(strings.ReplaceAll(flag, "-", "_"))
+	return fmt.Sprintf("%s_%s", EnvPrefix, envVarSuffix)
+}
+
+// buildHostCommand writes a new command to run in the host namespace
+// Rancher based need different pid
+func buildHostCommand(pid int, command []string) []string {
+
+	// From the container, we nsenter into the proper PID to run the hostCommand.
+	// For this, kured daemonset need to be configured with hostPID:true and privileged:true
+	cmd := []string{"/usr/bin/nsenter", fmt.Sprintf("-m/proc/%d/ns/mnt", pid), "--"}
+	cmd = append(cmd, command...)
 	return cmd
 }
 
-func sentinelExists() bool {
-	// Relies on hostPID:true and privileged:true to enter host mount space
-	sentinelCmd := newCommand("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "--", "/usr/bin/test", "-f", rebootSentinel)
-	if err := sentinelCmd.Run(); err != nil {
+func rebootRequired(sentinelCommand []string) bool {
+	cmd := util.NewCommand(sentinelCommand[0], sentinelCommand[1:]...)
+	if err := cmd.Run(); err != nil {
 		switch err := err.(type) {
 		case *exec.ExitError:
 			// We assume a non-zero exit code means 'reboot not required', but of course
@@ -142,6 +335,9 @@ func sentinelExists() bool {
 			// went wrong during its execution. In that case, not entering a reboot loop
 			// is the right thing to do, and we are logging stdout/stderr of the command
 			// so it should be obvious what is wrong.
+			if cmd.ProcessState.ExitCode() != 1 {
+				log.Warnf("sentinel command ended with unexpected exit code: %v", cmd.ProcessState.ExitCode())
+			}
 			return false
 		default:
 			// Something was grossly misconfigured, such as the command path being wrong.
@@ -151,35 +347,57 @@ func sentinelExists() bool {
 	return true
 }
 
-func rebootRequired() bool {
-	if sentinelExists() {
-		log.Infof("Reboot required")
+// RebootBlocker interface should be implemented by types
+// to know if their instantiations should block a reboot
+type RebootBlocker interface {
+	isBlocked() bool
+}
+
+// PrometheusBlockingChecker contains info for connecting
+// to prometheus, and can give info about whether a reboot should be blocked
+type PrometheusBlockingChecker struct {
+	// prometheusClient to make prometheus-go-client and api config available
+	// into the PrometheusBlockingChecker struct
+	promClient *alerts.PromClient
+	// regexp used to get alerts
+	filter *regexp.Regexp
+	// bool to indicate if only firing alerts should be considered
+	firingOnly bool
+	// bool to indicate that we're only blocking on alerts which match the filter
+	filterMatchOnly bool
+}
+
+// KubernetesBlockingChecker contains info for connecting
+// to k8s, and can give info about whether a reboot should be blocked
+type KubernetesBlockingChecker struct {
+	// client used to contact kubernetes API
+	client   *kubernetes.Clientset
+	nodename string
+	// lised used to filter pods (podSelector)
+	filter []string
+}
+
+func (pb PrometheusBlockingChecker) isBlocked() bool {
+	alertNames, err := pb.promClient.ActiveAlerts(pb.filter, pb.firingOnly, pb.filterMatchOnly)
+	if err != nil {
+		log.Warnf("Reboot blocked: prometheus query error: %v", err)
+		return true
+	}
+	count := len(alertNames)
+	if count > 10 {
+		alertNames = append(alertNames[:10], "...")
+	}
+	if count > 0 {
+		log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
 		return true
 	}
-	log.Infof("Reboot not required")
 	return false
 }
 
-func rebootBlocked(client *kubernetes.Clientset, nodeID string) bool {
-	if prometheusURL != "" {
-		alertNames, err := alerts.PrometheusActiveAlerts(prometheusURL, alertFilter)
-		if err != nil {
-			log.Warnf("Reboot blocked: prometheus query error: %v", err)
-			return true
-		}
-		count := len(alertNames)
-		if count > 10 {
-			alertNames = append(alertNames[:10], "...")
-		}
-		if count > 0 {
-			log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
-			return true
-		}
-	}
-
-	fieldSelector := fmt.Sprintf("spec.nodeName=%s", nodeID)
-	for _, labelSelector := range podSelectors {
-		podList, err := client.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
+func (kb KubernetesBlockingChecker) isBlocked() bool {
+	fieldSelector := fmt.Sprintf("spec.nodeName=%s,status.phase!=Succeeded,status.phase!=Failed,status.phase!=Unknown", kb.nodename)
+	for _, labelSelector := range kb.filter {
+		podList, err := kb.client.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
 			LabelSelector: labelSelector,
 			FieldSelector: fieldSelector,
 			Limit:         10})
@@ -200,12 +418,26 @@ func rebootBlocked(client *kubernetes.Clientset, nodeID string) bool {
 			return true
 		}
 	}
-
 	return false
 }
 
-func holding(lock *daemonsetlock.DaemonSetLock, metadata interface{}) bool {
-	holding, err := lock.Test(metadata)
+func rebootBlocked(blockers ...RebootBlocker) bool {
+	for _, blocker := range blockers {
+		if blocker.isBlocked() {
+			return true
+		}
+	}
+	return false
+}
+
+func holding(lock *daemonsetlock.DaemonSetLock, metadata interface{}, isMultiLock bool) bool {
+	var holding bool
+	var err error
+	if isMultiLock {
+		holding, err = lock.TestMultiple()
+	} else {
+		holding, err = lock.Test(metadata)
+	}
 	if err != nil {
 		log.Fatalf("Error testing lock: %v", err)
 	}
@@ -215,8 +447,17 @@ func holding(lock *daemonsetlock.DaemonSetLock, metadata interface{}) bool {
 	return holding
 }
 
-func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}, TTL time.Duration) bool {
-	holding, holder, err := lock.Acquire(metadata, TTL)
+func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}, TTL time.Duration, maxOwners int) bool {
+	var holding bool
+	var holder string
+	var err error
+	if maxOwners > 1 {
+		var holders []string
+		holding, holders, err = lock.AcquireMultiple(metadata, TTL, maxOwners)
+		holder = strings.Join(holders, ",")
+	} else {
+		holding, holder, err = lock.Acquire(metadata, TTL)
+	}
 	switch {
 	case err != nil:
 		log.Fatalf("Error acquiring lock: %v", err)
@@ -230,74 +471,94 @@ func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}, TTL time.D
 	}
 }
 
-func release(lock *daemonsetlock.DaemonSetLock) {
+func throttle(releaseDelay time.Duration) {
+	if releaseDelay > 0 {
+		log.Infof("Delaying lock release by %v", releaseDelay)
+		time.Sleep(releaseDelay)
+	}
+}
+
+func release(lock *daemonsetlock.DaemonSetLock, isMultiLock bool) {
 	log.Infof("Releasing lock")
-	if err := lock.Release(); err != nil {
+
+	var err error
+	if isMultiLock {
+		err = lock.ReleaseMultiple()
+	} else {
+		err = lock.Release()
+	}
+	if err != nil {
 		log.Fatalf("Error releasing lock: %v", err)
 	}
 }
 
-func drain(client *kubernetes.Clientset, node *v1.Node) {
+func drain(client *kubernetes.Clientset, node *v1.Node) error {
 	nodename := node.GetName()
 
+	if preRebootNodeLabels != nil {
+		updateNodeLabels(client, node, preRebootNodeLabels)
+	}
+
+	if drainDelay > 0 {
+		log.Infof("Delaying drain for %v", drainDelay)
+		time.Sleep(drainDelay)
+	}
+
 	log.Infof("Draining node %s", nodename)
 
-	if slackHookURL != "" {
-		if err := slack.NotifyDrain(slackHookURL, slackUsername, slackChannel, messageTemplateDrain, nodename); err != nil {
-			log.Warnf("Error notifying slack: %v", err)
+	if notifyURL != "" {
+		if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateDrain, nodename)); err != nil {
+			log.Warnf("Error notifying: %v", err)
 		}
 	}
 
 	drainer := &kubectldrain.Helper{
-		Client:              client,
-		GracePeriodSeconds:  -1,
-		Force:               true,
-		DeleteLocalData:     true,
-		IgnoreAllDaemonSets: true,
-		ErrOut:              os.Stderr,
-		Out:                 os.Stdout,
+		Client:                          client,
+		Ctx:                             context.Background(),
+		GracePeriodSeconds:              drainGracePeriod,
+		PodSelector:                     drainPodSelector,
+		SkipWaitForDeleteTimeoutSeconds: skipWaitForDeleteTimeoutSeconds,
+		Force:                           true,
+		DeleteEmptyDirData:              true,
+		IgnoreAllDaemonSets:             true,
+		ErrOut:                          os.Stderr,
+		Out:                             os.Stdout,
+		Timeout:                         drainTimeout,
 	}
+
 	if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil {
-		log.Fatal("Error cordonning %s: %v", nodename, err)
+		log.Errorf("Error cordonning %s: %v", nodename, err)
+		return err
 	}
 
 	if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil {
-		log.Fatal("Error draining %s: %v", nodename, err)
+		log.Errorf("Error draining %s: %v", nodename, err)
+		return err
 	}
+	return nil
 }
 
-func uncordon(client *kubernetes.Clientset, node *v1.Node) {
+func uncordon(client *kubernetes.Clientset, node *v1.Node) error {
 	nodename := node.GetName()
 	log.Infof("Uncordoning node %s", nodename)
 	drainer := &kubectldrain.Helper{
 		Client: client,
 		ErrOut: os.Stderr,
 		Out:    os.Stdout,
+		Ctx:    context.Background(),
 	}
 	if err := kubectldrain.RunCordonOrUncordon(drainer, node, false); err != nil {
-		log.Fatal("Error uncordonning %s: %v", nodename, err)
+		log.Fatalf("Error uncordonning %s: %v", nodename, err)
+		return err
+	} else if postRebootNodeLabels != nil {
+		updateNodeLabels(client, node, postRebootNodeLabels)
 	}
+	return nil
 }
 
-func commandReboot(nodeID string) {
-	log.Infof("Commanding reboot for node: %s", nodeID)
-
-	if slackHookURL != "" {
-		if err := slack.NotifyReboot(slackHookURL, slackUsername, slackChannel, messageTemplateReboot, nodeID); err != nil {
-			log.Warnf("Error notifying slack: %v", err)
-		}
-	}
-
-	// Relies on hostPID:true and privileged:true to enter host mount space
-	rebootCmd := newCommand("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/bin/systemctl", "reboot")
-	if err := rebootCmd.Run(); err != nil {
-		log.Fatalf("Error invoking reboot command: %v", err)
-	}
-}
-
-func maintainRebootRequiredMetric(nodeID string) {
+func maintainRebootRequiredMetric(nodeID string, sentinelCommand []string) {
 	for {
-		if sentinelExists() {
+		if rebootRequired(sentinelCommand) {
 			rebootRequiredGauge.WithLabelValues(nodeID).Set(1)
 		} else {
 			rebootRequiredGauge.WithLabelValues(nodeID).Set(0)
@@ -311,7 +572,81 @@ type nodeMeta struct {
 	Unschedulable bool `json:"unschedulable"`
 }
 
-func rebootAsRequired(nodeID string, window *timewindow.TimeWindow, TTL time.Duration) {
+func addNodeAnnotations(client *kubernetes.Clientset, nodeID string, annotations map[string]string) error {
+	node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+	if err != nil {
+		log.Errorf("Error retrieving node object via k8s API: %s", err)
+		return err
+	}
+	for k, v := range annotations {
+		node.Annotations[k] = v
+		log.Infof("Adding node %s annotation: %s=%s", node.GetName(), k, v)
+	}
+
+	bytes, err := json.Marshal(node)
+	if err != nil {
+		log.Errorf("Error marshalling node object into JSON: %v", err)
+		return err
+	}
+
+	_, err = client.CoreV1().Nodes().Patch(context.TODO(), node.GetName(), types.StrategicMergePatchType, bytes, metav1.PatchOptions{})
+	if err != nil {
+		var annotationsErr string
+		for k, v := range annotations {
+			annotationsErr += fmt.Sprintf("%s=%s ", k, v)
+		}
+		log.Errorf("Error adding node annotations %s via k8s API: %v", annotationsErr, err)
+		return err
+	}
+	return nil
+}
+
+func deleteNodeAnnotation(client *kubernetes.Clientset, nodeID, key string) error {
+	log.Infof("Deleting node %s annotation %s", nodeID, key)
+
+	// JSON Patch takes as path input a JSON Pointer, defined in RFC6901
+	// So we replace all instances of "/" with "~1" as per:
+	// https://tools.ietf.org/html/rfc6901#section-3
+	patch := []byte(fmt.Sprintf("[{\"op\":\"remove\",\"path\":\"/metadata/annotations/%s\"}]", strings.ReplaceAll(key, "/", "~1")))
+	_, err := client.CoreV1().Nodes().Patch(context.TODO(), nodeID, types.JSONPatchType, patch, metav1.PatchOptions{})
+	if err != nil {
+		log.Errorf("Error deleting node annotation %s via k8s API: %v", key, err)
+		return err
+	}
+	return nil
+}
+
+func updateNodeLabels(client *kubernetes.Clientset, node *v1.Node, labels []string) {
+	labelsMap := make(map[string]string)
+	for _, label := range labels {
+		k := strings.Split(label, "=")[0]
+		v := strings.Split(label, "=")[1]
+		labelsMap[k] = v
+		log.Infof("Updating node %s label: %s=%s", node.GetName(), k, v)
+	}
+
+	bytes, err := json.Marshal(map[string]interface{}{
+		"metadata": map[string]interface{}{
+			"labels": labelsMap,
+		},
+	})
+	if err != nil {
+		log.Fatalf("Error marshalling node object into JSON: %v", err)
+	}
+
+	_, err = client.CoreV1().Nodes().Patch(context.TODO(), node.GetName(), types.StrategicMergePatchType, bytes, metav1.PatchOptions{})
+	if err != nil {
+		var labelsErr string
+		for _, label := range labels {
+			k := strings.Split(label, "=")[0]
+			v := strings.Split(label, "=")[1]
+			labelsErr += fmt.Sprintf("%s=%s ", k, v)
+		}
+		log.Errorf("Error updating node labels %s via k8s API: %v", labelsErr, err)
+	}
+}
+
+func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []string, window *timewindow.TimeWindow, TTL time.Duration, releaseDelay time.Duration) {
 	config, err := rest.InClusterConfig()
 	if err != nil {
 		log.Fatal(err)
@@ -325,45 +660,180 @@ func rebootAsRequired(nodeID string, window *timewindow.TimeWindow, TTL time.Dur
 	lock := daemonsetlock.New(client, nodeID, dsNamespace, dsName, lockAnnotation)
 
 	nodeMeta := nodeMeta{}
-	if holding(lock, &nodeMeta) {
-		if !nodeMeta.Unschedulable {
+	source := rand.NewSource(time.Now().UnixNano())
+	tick := delaytick.New(source, 1*time.Minute)
+	for range tick {
+		if holding(lock, &nodeMeta, concurrency > 1) {
 			node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
 			if err != nil {
-				log.Fatal(err)
+				log.Errorf("Error retrieving node object via k8s API: %v", err)
+				continue
 			}
-			uncordon(client, node)
+			if !nodeMeta.Unschedulable {
+				err = uncordon(client, node)
+				if err != nil {
+					log.Errorf("Unable to uncordon %s: %v, will continue to hold lock and retry uncordon", node.GetName(), err)
+					continue
+				} else {
+					if notifyURL != "" {
+						if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateUncordon, nodeID)); err != nil {
+							log.Warnf("Error notifying: %v", err)
+						}
+					}
+				}
+			}
+			// If we're holding the lock we know we've tried, in a prior run, to reboot
+			// So (1) we want to confirm that the reboot succeeded practically ( !rebootRequired() )
+			// And (2) check if we previously annotated the node that it was in the process of being rebooted,
+			// And finally (3) if it has that annotation, to delete it.
+			// This indicates to other node tools running on the cluster that this node may be a candidate for maintenance
+			if annotateNodes && !rebootRequired(sentinelCommand) {
+				if _, ok := node.Annotations[KuredRebootInProgressAnnotation]; ok {
+					err := deleteNodeAnnotation(client, nodeID, KuredRebootInProgressAnnotation)
+					if err != nil {
+						continue
+					}
+				}
+			}
+			throttle(releaseDelay)
+			release(lock, concurrency > 1)
+			break
+		} else {
+			break
 		}
-		release(lock)
 	}
 
-	source := rand.NewSource(time.Now().UnixNano())
-	tick := delaytick.New(source, period)
-	for range tick {
-		if window.Contains(time.Now()) && rebootRequired() && !rebootBlocked(client, nodeID) {
-			node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
-			if err != nil {
-				log.Fatal(err)
-			}
-			nodeMeta.Unschedulable = node.Spec.Unschedulable
+	preferNoScheduleTaint := taints.New(client, nodeID, preferNoScheduleTaintName, v1.TaintEffectPreferNoSchedule)
 
-			if acquire(lock, &nodeMeta, TTL) {
-				if !nodeMeta.Unschedulable {
-					drain(client, node)
-				}
-				commandReboot(nodeID)
-				for {
-					log.Infof("Waiting for reboot")
-					time.Sleep(time.Minute)
+	// Remove taint immediately during startup to quickly allow scheduling again.
+	if !rebootRequired(sentinelCommand) {
+		preferNoScheduleTaint.Disable()
+	}
+
+	// instantiate prometheus client
+	promClient, err := alerts.NewPromClient(papi.Config{Address: prometheusURL})
+	if err != nil {
+		log.Fatal("Unable to create prometheus client: ", err)
+	}
+
+	source = rand.NewSource(time.Now().UnixNano())
+	tick = delaytick.New(source, period)
+	for range tick {
+		if !window.Contains(time.Now()) {
+			// Remove taint outside the reboot time window to allow for normal operation.
+			preferNoScheduleTaint.Disable()
+			continue
+		}
+
+		if !rebootRequired(sentinelCommand) {
+			log.Infof("Reboot not required")
+			preferNoScheduleTaint.Disable()
+			continue
+		}
+
+		node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+		if err != nil {
+			log.Fatalf("Error retrieving node object via k8s API: %v", err)
+		}
+		nodeMeta.Unschedulable = node.Spec.Unschedulable
+
+		var timeNowString string
+		if annotateNodes {
+			if _, ok := node.Annotations[KuredRebootInProgressAnnotation]; !ok {
+				timeNowString = time.Now().Format(time.RFC3339)
+				// Annotate this node to indicate that "I am going to be rebooted!"
+				// so that other node maintenance tools running on the cluster are aware that this node is in the process of a "state transition"
+				annotations := map[string]string{KuredRebootInProgressAnnotation: timeNowString}
+				// & annotate this node with a timestamp so that other node maintenance tools know how long it's been since this node has been marked for reboot
+				annotations[KuredMostRecentRebootNeededAnnotation] = timeNowString
+				err := addNodeAnnotations(client, nodeID, annotations)
+				if err != nil {
+					continue
 				}
 			}
 		}
+
+		var blockCheckers []RebootBlocker
+		if prometheusURL != "" {
+			blockCheckers = append(blockCheckers, PrometheusBlockingChecker{promClient: promClient, filter: alertFilter, firingOnly: alertFiringOnly, filterMatchOnly: alertFilterMatchOnly})
+		}
+		if podSelectors != nil {
+			blockCheckers = append(blockCheckers, KubernetesBlockingChecker{client: client, nodename: nodeID, filter: podSelectors})
+		}
+
+		var rebootRequiredBlockCondition string
+		if rebootBlocked(blockCheckers...) {
+			rebootRequiredBlockCondition = ", but blocked at this time"
+			continue
+		}
+		log.Infof("Reboot required%s", rebootRequiredBlockCondition)
+
+		if !holding(lock, &nodeMeta, concurrency > 1) && !acquire(lock, &nodeMeta, TTL, concurrency) {
+			// Prefer to not schedule pods onto this node to avoid draing the same pod multiple times.
+			preferNoScheduleTaint.Enable()
+			continue
+		}
+
+		err = drain(client, node)
+		if err != nil {
+			if !forceReboot {
+				log.Errorf("Unable to cordon or drain %s: %v, will release lock and retry cordon and drain before rebooting when lock is next acquired", node.GetName(), err)
+				release(lock, concurrency > 1)
+				log.Infof("Performing a best-effort uncordon after failed cordon and drain")
+				uncordon(client, node)
+				continue
+			}
+		}
+
+		if rebootDelay > 0 {
+			log.Infof("Delaying reboot for %v", rebootDelay)
+			time.Sleep(rebootDelay)
+		}
+
+		if notifyURL != "" {
+			if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateReboot, nodeID)); err != nil {
+				log.Warnf("Error notifying: %v", err)
+			}
+		}
+
+		booter.Reboot()
+		for {
+			log.Infof("Waiting for reboot")
+			time.Sleep(time.Minute)
+		}
 	}
 }
 
+// buildSentinelCommand creates the shell command line which will need wrapping to escape
+// the container boundaries
+func buildSentinelCommand(rebootSentinelFile string, rebootSentinelCommand string) []string {
+	if rebootSentinelCommand != "" {
+		cmd, err := shlex.Split(rebootSentinelCommand)
+		if err != nil {
+			log.Fatalf("Error parsing provided sentinel command: %v", err)
+		}
+		return cmd
+	}
+	return []string{"test", "-f", rebootSentinelFile}
+}
+
+// parseRebootCommand creates the shell command line which will need wrapping to escape
+// the container boundaries
+func parseRebootCommand(rebootCommand string) []string {
+	command, err := shlex.Split(rebootCommand)
+	if err != nil {
+		log.Fatalf("Error parsing provided reboot command: %v", err)
+	}
+	return command
+}
+
 func root(cmd *cobra.Command, args []string) {
+	if logFormat == "json" {
+		log.SetFormatter(&log.JSONFormatter{})
+	}
+
 	log.Infof("Kubernetes Reboot Daemon: %s", version)
 
-	nodeID := os.Getenv("KURED_NODE_ID")
 	if nodeID == "" {
 		log.Fatal("KURED_NODE_ID environment variable required")
 	}
@@ -373,6 +843,9 @@ func root(cmd *cobra.Command, args []string) {
 		log.Fatalf("Failed to build time window: %v", err)
 	}
 
+	sentinelCommand := buildSentinelCommand(rebootSentinelFile, rebootSentinelCommand)
+	restartCommand := parseRebootCommand(rebootCommand)
+
 	log.Infof("Node ID: %s", nodeID)
 	log.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation)
 	if lockTTL > 0 {
@@ -380,13 +853,50 @@ func root(cmd *cobra.Command, args []string) {
 	} else {
 		log.Info("Lock TTL not set, lock will remain until being released")
 	}
-	log.Infof("Reboot Sentinel: %s every %v", rebootSentinel, period)
+	if lockReleaseDelay > 0 {
+		log.Infof("Lock release delay set, lock release will be delayed by: %v", lockReleaseDelay)
+	} else {
+		log.Info("Lock release delay not set, lock will be released immediately after rebooting")
+	}
+	log.Infof("PreferNoSchedule taint: %s", preferNoScheduleTaintName)
 	log.Infof("Blocking Pod Selectors: %v", podSelectors)
-	log.Infof("Reboot on: %v", window)
+	log.Infof("Reboot schedule: %v", window)
+	log.Infof("Reboot check command: %s every %v", sentinelCommand, period)
+	log.Infof("Concurrency: %v", concurrency)
+	log.Infof("Reboot method: %s", rebootMethod)
+	if rebootCommand == MethodCommand {
+		log.Infof("Reboot command: %s", restartCommand)
+	} else {
+		log.Infof("Reboot signal: %v", rebootSignal)
+	}
 
-	go rebootAsRequired(nodeID, window, lockTTL)
-	go maintainRebootRequiredMetric(nodeID)
+	if annotateNodes {
+		log.Infof("Will annotate nodes during kured reboot operations")
+	}
+
+	// To run those commands as it was the host, we'll use nsenter
+	// Relies on hostPID:true and privileged:true to enter host mount space
+	// PID set to 1, until we have a better discovery mechanism.
+	hostRestartCommand := buildHostCommand(1, restartCommand)
+
+	// Only wrap sentinel-command with nsenter, if a custom-command was configured, otherwise use the host-path mount
+	hostSentinelCommand := sentinelCommand
+	if rebootSentinelCommand != "" {
+		hostSentinelCommand = buildHostCommand(1, sentinelCommand)
+	}
+
+	var booter reboot.Reboot
+	if rebootMethod == MethodCommand {
+		booter = reboot.NewCommandReboot(nodeID, hostRestartCommand)
+	} else if rebootMethod == MethodSignal {
+		booter = reboot.NewSignalReboot(nodeID, rebootSignal)
+	} else {
+		log.Fatalf("Invalid reboot-method configured: %s", rebootMethod)
+	}
+
+	go rebootAsRequired(nodeID, booter, hostSentinelCommand, window, lockTTL, lockReleaseDelay)
+	go maintainRebootRequiredMetric(nodeID, hostSentinelCommand)
 
 	http.Handle("/metrics", promhttp.Handler())
-	log.Fatal(http.ListenAndServe(":8080", nil))
+	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", metricsHost, metricsPort), nil))
 }

cmd/kured/main_test.go

@@ -0,0 +1,310 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/kubereboot/kured/pkg/alerts"
+	assert "gotest.tools/v3/assert"
+
+	papi "github.com/prometheus/client_golang/api"
+)
+
+type BlockingChecker struct {
+	blocking bool
+}
+
+func (fbc BlockingChecker) isBlocked() bool {
+	return fbc.blocking
+}
+
+var _ RebootBlocker = BlockingChecker{}       // Verify that Type implements Interface.
+var _ RebootBlocker = (*BlockingChecker)(nil) // Verify that *Type implements Interface.
+
+func Test_flagCheck(t *testing.T) {
+	var cmd *cobra.Command
+	var args []string
+	slackHookURL = "https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"
+	expected := "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"
+	flagCheck(cmd, args)
+	if notifyURL != expected {
+		t.Errorf("Slack URL Parsing is wrong: expecting %s  but got %s\n", expected, notifyURL)
+	}
+
+	// validate that surrounding quotes are stripped
+	slackHookURL = "\"https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET\""
+	expected = "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"
+	flagCheck(cmd, args)
+	if notifyURL != expected {
+		t.Errorf("Slack URL Parsing is wrong: expecting %s  but got %s\n", expected, notifyURL)
+	}
+	slackHookURL = "'https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET'"
+	expected = "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"
+	flagCheck(cmd, args)
+	if notifyURL != expected {
+		t.Errorf("Slack URL Parsing is wrong: expecting %s  but got %s\n", expected, notifyURL)
+	}
+	slackHookURL = ""
+	notifyURL = "\"teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com\""
+	expected = "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com"
+	flagCheck(cmd, args)
+	if notifyURL != expected {
+		t.Errorf("notifyURL Parsing is wrong: expecting %s  but got %s\n", expected, notifyURL)
+	}
+	notifyURL = "'teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com'"
+	expected = "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com"
+	flagCheck(cmd, args)
+	if notifyURL != expected {
+		t.Errorf("notifyURL Parsing is wrong: expecting %s  but got %s\n", expected, notifyURL)
+	}
+}
+
+func Test_stripQuotes(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "string with no surrounding quotes is unchanged",
+			input:    "Hello, world!",
+			expected: "Hello, world!",
+		},
+		{
+			name:     "string with surrounding double quotes should strip quotes",
+			input:    "\"Hello, world!\"",
+			expected: "Hello, world!",
+		},
+		{
+			name:     "string with surrounding single quotes should strip quotes",
+			input:    "'Hello, world!'",
+			expected: "Hello, world!",
+		},
+		{
+			name:     "string with unbalanced surrounding quotes is unchanged",
+			input:    "'Hello, world!\"",
+			expected: "'Hello, world!\"",
+		},
+		{
+			name:     "string with length of one is unchanged",
+			input:    "'",
+			expected: "'",
+		},
+		{
+			name:     "string with length of zero is unchanged",
+			input:    "",
+			expected: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := stripQuotes(tt.input); !reflect.DeepEqual(got, tt.expected) {
+				t.Errorf("stripQuotes() = %v, expected %v", got, tt.expected)
+			}
+		})
+	}
+}
+
+func Test_rebootBlocked(t *testing.T) {
+	noCheckers := []RebootBlocker{}
+	nonblockingChecker := BlockingChecker{blocking: false}
+	blockingChecker := BlockingChecker{blocking: true}
+
+	// Instantiate a prometheusClient with a broken_url
+	promClient, err := alerts.NewPromClient(papi.Config{Address: "broken_url"})
+	if err != nil {
+		log.Fatal("Can't create prometheusClient: ", err)
+	}
+	brokenPrometheusClient := PrometheusBlockingChecker{promClient: promClient, filter: nil, firingOnly: false}
+
+	type args struct {
+		blockers []RebootBlocker
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Do not block on no blocker defined",
+			args: args{blockers: noCheckers},
+			want: false,
+		},
+		{
+			name: "Ensure a blocker blocks",
+			args: args{blockers: []RebootBlocker{blockingChecker}},
+			want: true,
+		},
+		{
+			name: "Ensure a non-blocker doesn't block",
+			args: args{blockers: []RebootBlocker{nonblockingChecker}},
+			want: false,
+		},
+		{
+			name: "Ensure one blocker is enough to block",
+			args: args{blockers: []RebootBlocker{nonblockingChecker, blockingChecker}},
+			want: true,
+		},
+		{
+			name: "Do block on error contacting prometheus API",
+			args: args{blockers: []RebootBlocker{brokenPrometheusClient}},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := rebootBlocked(tt.args.blockers...); got != tt.want {
+				t.Errorf("rebootBlocked() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_buildHostCommand(t *testing.T) {
+	type args struct {
+		pid     int
+		command []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "Ensure command will run with nsenter",
+			args: args{pid: 1, command: []string{"ls", "-Fal"}},
+			want: []string{"/usr/bin/nsenter", "-m/proc/1/ns/mnt", "--", "ls", "-Fal"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := buildHostCommand(tt.args.pid, tt.args.command); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("buildHostCommand() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_buildSentinelCommand(t *testing.T) {
+	type args struct {
+		rebootSentinelFile    string
+		rebootSentinelCommand string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "Ensure a sentinelFile generates a shell 'test' command with the right file",
+			args: args{
+				rebootSentinelFile:    "/test1",
+				rebootSentinelCommand: "",
+			},
+			want: []string{"test", "-f", "/test1"},
+		},
+		{
+			name: "Ensure a sentinelCommand has priority over a sentinelFile if both are provided (because sentinelFile is always provided)",
+			args: args{
+				rebootSentinelFile:    "/test1",
+				rebootSentinelCommand: "/sbin/reboot-required -r",
+			},
+			want: []string{"/sbin/reboot-required", "-r"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := buildSentinelCommand(tt.args.rebootSentinelFile, tt.args.rebootSentinelCommand); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("buildSentinelCommand() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_parseRebootCommand(t *testing.T) {
+	type args struct {
+		rebootCommand string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "Ensure a reboot command is properly parsed",
+			args: args{
+				rebootCommand: "/sbin/systemctl reboot",
+			},
+			want: []string{"/sbin/systemctl", "reboot"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := parseRebootCommand(tt.args.rebootCommand); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseRebootCommand() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_rebootRequired(t *testing.T) {
+	type args struct {
+		sentinelCommand []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Ensure rc = 0 means reboot required",
+			args: args{
+				sentinelCommand: []string{"true"},
+			},
+			want: true,
+		},
+		{
+			name: "Ensure rc != 0 means reboot NOT required",
+			args: args{
+				sentinelCommand: []string{"false"},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := rebootRequired(tt.args.sentinelCommand); got != tt.want {
+				t.Errorf("rebootRequired() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_rebootRequired_fatals(t *testing.T) {
+	cases := []struct {
+		param       []string
+		expectFatal bool
+	}{
+		{
+			param:       []string{"true"},
+			expectFatal: false,
+		},
+		{
+			param:       []string{"./babar"},
+			expectFatal: true,
+		},
+	}
+
+	defer func() { log.StandardLogger().ExitFunc = nil }()
+	var fatal bool
+	log.StandardLogger().ExitFunc = func(int) { fatal = true }
+
+	for _, c := range cases {
+		fatal = false
+		rebootRequired(c.param)
+		assert.Equal(t, c.expectFatal, fatal)
+	}
+
+}

go.mod

@@ -1,14 +1,109 @@
-module github.com/weaveworks/kured
+module github.com/kubereboot/kured
 
-go 1.15
+go 1.20
+
+replace golang.org/x/net => golang.org/x/net v0.17.0
+
+replace github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 v3.10.2
 
 require (
-	github.com/prometheus/client_golang v1.8.0
-	github.com/prometheus/common v0.15.0
-	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/cobra v1.1.1
-	k8s.io/api v0.19.4
-	k8s.io/apimachinery v0.19.4
-	k8s.io/client-go v0.19.4
-	k8s.io/kubectl v0.19.4
+	github.com/containrrr/shoutrrr v0.8.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/google/uuid v1.4.0 // indirect
+	github.com/prometheus/client_golang v1.18.0
+	github.com/prometheus/common v0.46.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.8.0
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.18.2
+	github.com/stretchr/testify v1.8.4
+	gotest.tools/v3 v3.5.1
+	k8s.io/api v0.28.4
+	k8s.io/apimachinery v0.28.4
+	k8s.io/client-go v0.28.4
+	k8s.io/kubectl v0.28.4
+)
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
+	github.com/fatih/color v1.15.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/btree v1.0.1 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/xlab/treeprint v1.2.0 // indirect
+	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/oauth2 v0.16.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/cli-runtime v0.28.4 // indirect
+	k8s.io/component-base v0.28.4 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

kured-ds-signal.yaml

@@ -0,0 +1,100 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kured
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kured # Must match `--ds-name`
+  namespace: kube-system # Must match `--ds-namespace`
+spec:
+  selector:
+    matchLabels:
+      name: kured
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        name: kured
+    spec:
+      serviceAccountName: kured
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      hostPID: true # Facilitate entering the host mount namespace via init
+      restartPolicy: Always
+      volumes:
+        - name: sentinel
+          hostPath:
+            path: /var/run
+            type: Directory
+      containers:
+        - name: kured
+          # If you find yourself here wondering why there is no
+          # :latest tag on Docker Hub,see the FAQ in the README
+          image: ghcr.io/kubereboot/kured:1.15.0
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            privileged: false # Give permission to nsenter /proc/1/ns/mnt
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["*"]
+              add: ["CAP_KILL"]
+          ports:
+            - containerPort: 8080
+              name: metrics
+          env:
+            # Pass in the name of the node on which this pod is scheduled
+            # for use with drain/uncordon operations and lock acquisition
+            - name: KURED_NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /sentinel
+              name: sentinel
+              readOnly: true
+          command:
+            - /usr/bin/kured
+            - --reboot-sentinel=/sentinel/reboot-required
+            - --reboot-method=signal
+#            - --reboot-signal=39
+#            - --force-reboot=false
+#            - --drain-grace-period=-1
+#            - --skip-wait-for-delete-timeout=0
+#            - --drain-timeout=0
+#            - --period=1h
+#            - --ds-namespace=kube-system
+#            - --ds-name=kured
+#            - --lock-annotation=weave.works/kured-node-lock
+#            - --lock-ttl=0
+#            - --prometheus-url=http://prometheus.monitoring.svc.cluster.local
+#            - --alert-filter-regexp=^RebootRequired$
+#            - --alert-firing-only=false
+#            - --prefer-no-schedule-taint=""
+#            - --reboot-sentinel-command=""
+#            - --slack-hook-url=https://hooks.slack.com/...
+#            - --slack-username=prod
+#            - --slack-channel=alerting
+#            - --notify-url="" # See also shoutrrr url format
+#            - --message-template-drain=Draining node %s
+#            - --message-template-reboot=Rebooting node %s
+#            - --message-template-uncordon=Node %s rebooted & uncordoned successfully!
+#            - --blocking-pod-selector=runtime=long,cost=expensive
+#            - --blocking-pod-selector=name=temperamental
+#            - --blocking-pod-selector=...
+#            - --reboot-days=sun,mon,tue,wed,thu,fri,sat
+#            - --reboot-delay=90s
+#            - --start-time=0:00
+#            - --end-time=23:59:59
+#            - --time-zone=UTC
+#            - --annotate-nodes=false
+#            - --lock-release-delay=30m
+#            - --log-format=text

kured-ds.yaml

@@ -8,14 +8,14 @@ metadata:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: kured            # Must match `--ds-name`
+  name: kured # Must match `--ds-name`
   namespace: kube-system # Must match `--ds-namespace`
 spec:
   selector:
     matchLabels:
       name: kured
   updateStrategy:
-   type: RollingUpdate
+    type: RollingUpdate
   template:
     metadata:
       labels:
@@ -23,18 +23,29 @@ spec:
     spec:
       serviceAccountName: kured
       tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
       hostPID: true # Facilitate entering the host mount namespace via init
       restartPolicy: Always
+      volumes:
+        - name: sentinel
+          hostPath:
+            path: /var/run
+            type: Directory
       containers:
         - name: kured
-          image: docker.io/weaveworks/kured
-                 # If you find yourself here wondering why there is no
-                 # :latest tag on Docker Hub,see the FAQ in the README
+          # If you find yourself here wondering why there is no
+          # :latest tag on Docker Hub,see the FAQ in the README
+          image: ghcr.io/kubereboot/kured:1.15.0
           imagePullPolicy: IfNotPresent
           securityContext:
             privileged: true # Give permission to nsenter /proc/1/ns/mnt
+            readOnlyRootFilesystem: true
+          ports:
+            - containerPort: 8080
+              name: metrics
           env:
             # Pass in the name of the node on which this pod is scheduled
             # for use with drain/uncordon operations and lock acquisition
@@ -42,24 +53,50 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /sentinel
+              name: sentinel
+              readOnly: true
           command:
             - /usr/bin/kured
-#            - --alert-filter-regexp=^RebootRequired$
-#            - --blocking-pod-selector=runtime=long,cost=expensive
-#            - --blocking-pod-selector=name=temperamental
-#            - --blocking-pod-selector=...
-#            - --ds-name=kured
-#            - --ds-namespace=kube-system
-#            - --end-time=23:59:59
-#            - --lock-annotation=weave.works/kured-node-lock
+            - --reboot-sentinel=/sentinel/reboot-required
+#            - --force-reboot=false
+#            - --drain-grace-period=-1
+#            - --skip-wait-for-delete-timeout=0
+#            - --drain-delay=0
+#            - --drain-timeout=0
+#            - --drain-pod-selector=""
 #            - --period=1h
+#            - --ds-namespace=kube-system
+#            - --ds-name=kured
+#            - --lock-annotation=weave.works/kured-node-lock
+#            - --lock-ttl=0
 #            - --prometheus-url=http://prometheus.monitoring.svc.cluster.local
-#            - --reboot-days=sun,mon,tue,wed,thu,fri,sat
-#            - --reboot-sentinel=/var/run/reboot-required
+#            - --alert-filter-regexp=^RebootRequired$
+#            - --alert-filter-match-only=false
+#            - --alert-firing-only=false
+#            - --prefer-no-schedule-taint=""
+#            - --reboot-sentinel-command=""
+#            - --reboot-method=command
+#            - --reboot-signal=39
 #            - --slack-hook-url=https://hooks.slack.com/...
 #            - --slack-username=prod
 #            - --slack-channel=alerting
+#            - --notify-url="" # See also shoutrrr url format
 #            - --message-template-drain=Draining node %s
-#            - --message-template-drain=Rebooting node %s
+#            - --message-template-reboot=Rebooting node %s
+#            - --message-template-uncordon=Node %s rebooted & uncordoned successfully!
+#            - --blocking-pod-selector=runtime=long,cost=expensive
+#            - --blocking-pod-selector=name=temperamental
+#            - --blocking-pod-selector=...
+#            - --reboot-days=sun,mon,tue,wed,thu,fri,sat
+#            - --reboot-delay=90s
 #            - --start-time=0:00
+#            - --end-time=23:59:59
 #            - --time-zone=UTC
+#            - --annotate-nodes=false
+#            - --lock-release-delay=30m
+#            - --log-format=text
+#            - --metrics-host=""
+#            - --metrics-port=8080
+#            - --concurrency=1

pkg/alerts/prometheus.go

@@ -7,22 +7,39 @@ import (
 	"sort"
 	"time"
 
-	"github.com/prometheus/client_golang/api"
+	papi "github.com/prometheus/client_golang/api"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/common/model"
 )
 
-// PrometheusActiveAlerts returns a list of names of active (e.g. pending or firing) alerts, filtered
-// by the supplied regexp.
-func PrometheusActiveAlerts(prometheusURL string, filter *regexp.Regexp) ([]string, error) {
-	client, err := api.NewClient(api.Config{Address: prometheusURL})
+// PromClient is a wrapper around the Prometheus Client interface and implements the api
+// This way, the PromClient can be instantiated with the configuration the Client needs, and
+// the ability to use the methods the api has, like Query and so on.
+type PromClient struct {
+	papi papi.Client
+	api  v1.API
+}
+
+// NewPromClient creates a new client to the Prometheus API.
+// It returns an error on any problem.
+func NewPromClient(conf papi.Config) (*PromClient, error) {
+	promClient, err := papi.NewClient(conf)
 	if err != nil {
 		return nil, err
 	}
+	client := PromClient{papi: promClient, api: v1.NewAPI(promClient)}
+	return &client, nil
+}
 
-	queryAPI := v1.NewAPI(client)
+// ActiveAlerts is a method of type PromClient, it returns a list of names of active alerts
+// (e.g. pending or firing), filtered by the supplied regexp or by the includeLabels query.
+// filter by regexp means when the regex finds the alert-name; the alert is exluded from the
+// block-list and will NOT block rebooting. query by includeLabel means,
+// if the query finds an alert, it will include it to the block-list and it WILL block rebooting.
+func (p *PromClient) ActiveAlerts(filter *regexp.Regexp, firingOnly, filterMatchOnly bool) ([]string, error) {
 
-	value, _, err := queryAPI.Query(context.Background(), "ALERTS", time.Now())
+	// get all alerts from prometheus
+	value, _, err := p.api.Query(context.Background(), "ALERTS", time.Now())
 	if err != nil {
 		return nil, err
 	}
@@ -32,7 +49,7 @@ func PrometheusActiveAlerts(prometheusURL string, filter *regexp.Regexp) ([]stri
 			activeAlertSet := make(map[string]bool)
 			for _, sample := range vector {
 				if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 {
-					if filter == nil || !filter.MatchString(string(alertName)) {
+					if matchesRegex(filter, string(alertName), filterMatchOnly) && (!firingOnly || sample.Metric["alertstate"] == "firing") {
 						activeAlertSet[string(alertName)] = true
 					}
 				}
@@ -42,7 +59,7 @@ func PrometheusActiveAlerts(prometheusURL string, filter *regexp.Regexp) ([]stri
 			for activeAlert := range activeAlertSet {
 				activeAlerts = append(activeAlerts, activeAlert)
 			}
-			sort.Sort(sort.StringSlice(activeAlerts))
+			sort.Strings(activeAlerts)
 
 			return activeAlerts, nil
 		}
@@ -50,3 +67,11 @@ func PrometheusActiveAlerts(prometheusURL string, filter *regexp.Regexp) ([]stri
 
 	return nil, fmt.Errorf("Unexpected value type: %v", value)
 }
+
+func matchesRegex(filter *regexp.Regexp, alertName string, filterMatchOnly bool) bool {
+	if filter == nil {
+		return true
+	}
+
+	return filter.MatchString(string(alertName)) == filterMatchOnly
+}

pkg/alerts/prometheus_test.go

@@ -0,0 +1,166 @@
+package alerts
+
+import (
+	"log"
+	"net/http"
+	"net/http/httptest"
+
+	"regexp"
+	"testing"
+
+	"github.com/prometheus/client_golang/api"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type MockResponse struct {
+	StatusCode int
+	Body       []byte
+}
+
+// MockServerProperties ties a mock response to a url and a method
+type MockServerProperties struct {
+	URI        string
+	HTTPMethod string
+	Response   MockResponse
+}
+
+// NewMockServer sets up a new MockServer with properties ad starts the server.
+func NewMockServer(props ...MockServerProperties) *httptest.Server {
+
+	handler := http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			for _, proc := range props {
+				_, err := w.Write(proc.Response.Body)
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+		})
+	return httptest.NewServer(handler)
+}
+
+func TestActiveAlerts(t *testing.T) {
+	responsebody := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"GatekeeperViolations","alertstate":"firing","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"PodCrashing-dev","alertstate":"firing","container":"deployment","instance":"1.2.3.4:8080","job":"kube-state-metrics","namespace":"dev","pod":"dev-deployment-78dcbmf25v","severity":"critical","team":"dev"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"PodRestart-dev","alertstate":"firing","container":"deployment","instance":"1.2.3.4:1234","job":"kube-state-metrics","namespace":"qa","pod":"qa-job-deployment-78dcbmf25v","severity":"warning","team":"qa"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"PrometheusTargetDown","alertstate":"firing","job":"kubernetes-pods","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"ScheduledRebootFailing","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`
+	addr := "http://localhost:10001"
+
+	for _, tc := range []struct {
+		it              string
+		rFilter         string
+		respBody        string
+		aName           string
+		wantN           int
+		firingOnly      bool
+		filterMatchOnly bool
+	}{
+		{
+			it:              "should return no active alerts",
+			respBody:        responsebody,
+			rFilter:         "",
+			wantN:           0,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should return a subset of all alerts",
+			respBody:        responsebody,
+			rFilter:         "Pod",
+			wantN:           3,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should return a subset of all alerts",
+			respBody:        responsebody,
+			rFilter:         "Gatekeeper",
+			wantN:           1,
+			firingOnly:      false,
+			filterMatchOnly: true,
+		},
+		{
+			it:              "should return all active alerts by regex",
+			respBody:        responsebody,
+			rFilter:         "*",
+			wantN:           5,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should return all active alerts by regex filter",
+			respBody:        responsebody,
+			rFilter:         "*",
+			wantN:           5,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should return only firing alerts if firingOnly is true",
+			respBody:        responsebody,
+			rFilter:         "*",
+			wantN:           4,
+			firingOnly:      true,
+			filterMatchOnly: false,
+		},
+
+		{
+			it:              "should return ScheduledRebootFailing active alerts",
+			respBody:        `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"ScheduledRebootFailing","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`,
+			aName:           "ScheduledRebootFailing",
+			rFilter:         "*",
+			wantN:           1,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should not return an active alert if RebootRequired is firing (regex filter)",
+			respBody:        `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"RebootRequired","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`,
+			rFilter:         "RebootRequired",
+			wantN:           0,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should not return an active alert if RebootRequired is firing (regex filter)",
+			respBody:        `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"RebootRequired","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`,
+			rFilter:         "RebootRequired",
+			wantN:           1,
+			firingOnly:      false,
+			filterMatchOnly: true,
+		},
+	} {
+		// Start mockServer
+		mockServer := NewMockServer(MockServerProperties{
+			URI:        addr,
+			HTTPMethod: http.MethodPost,
+			Response: MockResponse{
+				Body: []byte(tc.respBody),
+			},
+		})
+		// Close mockServer after all connections are gone
+		defer mockServer.Close()
+
+		t.Run(tc.it, func(t *testing.T) {
+
+			// regex filter
+			regex, _ := regexp.Compile(tc.rFilter)
+
+			// instantiate the prometheus client with the mockserver-address
+			p, err := NewPromClient(api.Config{Address: mockServer.URL})
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			result, err := p.ActiveAlerts(regex, tc.firingOnly, tc.filterMatchOnly)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			// assert
+			assert.Equal(t, tc.wantN, len(result), "expected amount of alerts %v, got %v", tc.wantN, len(result))
+
+			if tc.aName != "" {
+				assert.Equal(t, tc.aName, result[0], "expected active alert %v, got %v", tc.aName, result[0])
+			}
+		})
+	}
+}

pkg/daemonsetlock/daemonsetlock.go

@@ -6,11 +6,18 @@ import (
 	"fmt"
 	"time"
 
+	v1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 )
 
+const (
+	k8sAPICallRetrySleep   = 5 * time.Second // How much time to wait in between retrying a k8s API call
+	k8sAPICallRetryTimeout = 5 * time.Minute // How long to wait until we determine that the k8s API is definitively unavailable
+)
+
 // DaemonSetLock holds all necessary information to do actions
 // on the kured ds which holds lock info through annotations.
 type DaemonSetLock struct {
@@ -28,17 +35,22 @@ type lockAnnotationValue struct {
 	TTL      time.Duration `json:"TTL"`
 }
 
+type multiLockAnnotationValue struct {
+	MaxOwners       int                   `json:"maxOwners"`
+	LockAnnotations []lockAnnotationValue `json:"locks"`
+}
+
 // New creates a daemonsetLock object containing the necessary data for follow up k8s requests
 func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation string) *DaemonSetLock {
 	return &DaemonSetLock{client, nodeID, namespace, name, annotation}
 }
 
 // Acquire attempts to annotate the kured daemonset with lock info from instantiated DaemonSetLock using client-go
-func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acquired bool, owner string, err error) {
+func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (bool, string, error) {
 	for {
-		ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{})
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
-			return false, "", err
+			return false, "", fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 		}
 
 		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
@@ -77,11 +89,97 @@ func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acqu
 	}
 }
 
+// AcquireMultiple creates and annotates the daemonset with a multiple owner lock
+func (dsl *DaemonSetLock) AcquireMultiple(metadata interface{}, TTL time.Duration, maxOwners int) (bool, []string, error) {
+	for {
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
+		if err != nil {
+			return false, []string{}, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
+		}
+
+		annotation := multiLockAnnotationValue{}
+		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
+		if exists {
+			if err := json.Unmarshal([]byte(valueString), &annotation); err != nil {
+				return false, []string{}, fmt.Errorf("error getting multi lock: %w", err)
+			}
+		}
+
+		lockPossible, newAnnotation := dsl.canAcquireMultiple(annotation, metadata, TTL, maxOwners)
+		if !lockPossible {
+			return false, nodeIDsFromMultiLock(newAnnotation), nil
+		}
+
+		if ds.ObjectMeta.Annotations == nil {
+			ds.ObjectMeta.Annotations = make(map[string]string)
+		}
+		newAnnotationBytes, err := json.Marshal(&newAnnotation)
+		if err != nil {
+			return false, []string{}, fmt.Errorf("error marshalling new annotation lock: %w", err)
+		}
+		ds.ObjectMeta.Annotations[dsl.annotation] = string(newAnnotationBytes)
+
+		_, err = dsl.client.AppsV1().DaemonSets(dsl.namespace).Update(context.Background(), ds, metav1.UpdateOptions{})
+		if err != nil {
+			if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == metav1.StatusReasonConflict {
+				time.Sleep(time.Second)
+				continue
+			} else {
+				return false, []string{}, fmt.Errorf("error updating daemonset with multi lock: %w", err)
+			}
+		}
+		return true, nodeIDsFromMultiLock(newAnnotation), nil
+	}
+}
+
+func nodeIDsFromMultiLock(annotation multiLockAnnotationValue) []string {
+	nodeIDs := make([]string, 0, len(annotation.LockAnnotations))
+	for _, nodeLock := range annotation.LockAnnotations {
+		nodeIDs = append(nodeIDs, nodeLock.NodeID)
+	}
+	return nodeIDs
+}
+
+func (dsl *DaemonSetLock) canAcquireMultiple(annotation multiLockAnnotationValue, metadata interface{}, TTL time.Duration, maxOwners int) (bool, multiLockAnnotationValue) {
+	newAnnotation := multiLockAnnotationValue{MaxOwners: maxOwners}
+	freeSpace := false
+	if annotation.LockAnnotations == nil || len(annotation.LockAnnotations) < maxOwners {
+		freeSpace = true
+		newAnnotation.LockAnnotations = annotation.LockAnnotations
+	} else {
+		for _, nodeLock := range annotation.LockAnnotations {
+			if ttlExpired(nodeLock.Created, nodeLock.TTL) {
+				freeSpace = true
+				continue
+			}
+			newAnnotation.LockAnnotations = append(
+				newAnnotation.LockAnnotations,
+				nodeLock,
+			)
+		}
+	}
+
+	if freeSpace {
+		newAnnotation.LockAnnotations = append(
+			newAnnotation.LockAnnotations,
+			lockAnnotationValue{
+				NodeID:   dsl.nodeID,
+				Metadata: metadata,
+				Created:  time.Now().UTC(),
+				TTL:      TTL,
+			},
+		)
+		return true, newAnnotation
+	}
+
+	return false, multiLockAnnotationValue{}
+}
+
 // Test attempts to check the kured daemonset lock status (existence, expiry) from instantiated DaemonSetLock using client-go
-func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) {
-	ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{})
+func (dsl *DaemonSetLock) Test(metadata interface{}) (bool, error) {
+	ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 	if err != nil {
-		return false, err
+		return false, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 	}
 
 	valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
@@ -99,12 +197,36 @@ func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) {
 	return false, nil
 }
 
+// TestMultiple attempts to check the kured daemonset lock status for multi locks
+func (dsl *DaemonSetLock) TestMultiple() (bool, error) {
+	ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
+	if err != nil {
+		return false, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
+	}
+
+	valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
+	if exists {
+		value := multiLockAnnotationValue{}
+		if err := json.Unmarshal([]byte(valueString), &value); err != nil {
+			return false, err
+		}
+
+		for _, nodeLock := range value.LockAnnotations {
+			if nodeLock.NodeID == dsl.nodeID && !ttlExpired(nodeLock.Created, nodeLock.TTL) {
+				return true, nil
+			}
+		}
+	}
+
+	return false, nil
+}
+
 // Release attempts to remove the lock data from the kured ds annotations using client-go
 func (dsl *DaemonSetLock) Release() error {
 	for {
-		ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{})
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
-			return err
+			return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 		}
 
 		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
@@ -137,6 +259,73 @@ func (dsl *DaemonSetLock) Release() error {
 	}
 }
 
+// ReleaseMultiple attempts to remove the lock data from the kured ds annotations using client-go
+func (dsl *DaemonSetLock) ReleaseMultiple() error {
+	for {
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
+		if err != nil {
+			return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
+		}
+
+		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
+		modified := false
+		value := multiLockAnnotationValue{}
+		if exists {
+			if err := json.Unmarshal([]byte(valueString), &value); err != nil {
+				return err
+			}
+
+			for idx, nodeLock := range value.LockAnnotations {
+				if nodeLock.NodeID == dsl.nodeID {
+					value.LockAnnotations = append(value.LockAnnotations[:idx], value.LockAnnotations[idx+1:]...)
+					modified = true
+					break
+				}
+			}
+		}
+
+		if !exists || !modified {
+			return fmt.Errorf("Lock not held")
+		}
+
+		newAnnotationBytes, err := json.Marshal(value)
+		if err != nil {
+			return fmt.Errorf("error marshalling new annotation on release: %v", err)
+		}
+		ds.ObjectMeta.Annotations[dsl.annotation] = string(newAnnotationBytes)
+
+		_, err = dsl.client.AppsV1().DaemonSets(dsl.namespace).Update(context.TODO(), ds, metav1.UpdateOptions{})
+		if err != nil {
+			if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == metav1.StatusReasonConflict {
+				// Something else updated the resource between us reading and writing - try again soon
+				time.Sleep(time.Second)
+				continue
+			} else {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+// GetDaemonSet returns the named DaemonSet resource from the DaemonSetLock's configured client
+func (dsl *DaemonSetLock) GetDaemonSet(sleep, timeout time.Duration) (*v1.DaemonSet, error) {
+	var ds *v1.DaemonSet
+	var lastError error
+	err := wait.PollImmediate(sleep, timeout, func() (bool, error) {
+		ctx, cancel := context.WithTimeout(context.Background(), timeout)
+		defer cancel()
+		if ds, lastError = dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(ctx, dsl.name, metav1.GetOptions{}); lastError != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Timed out trying to get daemonset %s in namespace %s: %v", dsl.name, dsl.namespace, lastError)
+	}
+	return ds, nil
+}
+
 func ttlExpired(created time.Time, ttl time.Duration) bool {
 	if ttl > 0 && time.Since(created) >= ttl {
 		return true

pkg/daemonsetlock/daemonsetlock_test.go

@@ -1,6 +1,8 @@
 package daemonsetlock
 
 import (
+	"reflect"
+	"sort"
 	"testing"
 	"time"
 )
@@ -26,3 +28,181 @@ func TestTtlExpired(t *testing.T) {
 		}
 	}
 }
+
+func multiLockAnnotationsAreEqualByNodes(src, dst multiLockAnnotationValue) bool {
+	srcNodes := []string{}
+	for _, srcLock := range src.LockAnnotations {
+		srcNodes = append(srcNodes, srcLock.NodeID)
+	}
+	sort.Strings(srcNodes)
+
+	dstNodes := []string{}
+	for _, dstLock := range dst.LockAnnotations {
+		dstNodes = append(dstNodes, dstLock.NodeID)
+	}
+	sort.Strings(dstNodes)
+
+	return reflect.DeepEqual(srcNodes, dstNodes)
+}
+
+func TestCanAcquireMultiple(t *testing.T) {
+	node1Name := "n1"
+	node2Name := "n2"
+	node3Name := "n3"
+	testCases := []struct {
+		name          string
+		daemonSetLock DaemonSetLock
+		maxOwners     int
+		current       multiLockAnnotationValue
+		desired       multiLockAnnotationValue
+		lockPossible  bool
+	}{
+		{
+			name: "empty_lock",
+			daemonSetLock: DaemonSetLock{
+				nodeID: node1Name,
+			},
+			maxOwners: 2,
+			current:   multiLockAnnotationValue{},
+			desired: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []lockAnnotationValue{
+					{NodeID: node1Name},
+				},
+			},
+			lockPossible: true,
+		},
+		{
+			name: "partial_lock",
+			daemonSetLock: DaemonSetLock{
+				nodeID: node1Name,
+			},
+			maxOwners: 2,
+			current: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []lockAnnotationValue{
+					{NodeID: node2Name},
+				},
+			},
+			desired: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []lockAnnotationValue{
+					{NodeID: node1Name},
+					{NodeID: node2Name},
+				},
+			},
+			lockPossible: true,
+		},
+		{
+			name: "full_lock",
+			daemonSetLock: DaemonSetLock{
+				nodeID: node1Name,
+			},
+			maxOwners: 2,
+			current: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []lockAnnotationValue{
+					{
+						NodeID:  node2Name,
+						Created: time.Now().UTC().Add(-1 * time.Minute),
+						TTL:     time.Hour,
+					},
+					{
+						NodeID:  node3Name,
+						Created: time.Now().UTC().Add(-1 * time.Minute),
+						TTL:     time.Hour,
+					},
+				},
+			},
+			desired: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []lockAnnotationValue{
+					{NodeID: node2Name},
+					{NodeID: node3Name},
+				},
+			},
+			lockPossible: false,
+		},
+		{
+			name: "full_with_one_expired_lock",
+			daemonSetLock: DaemonSetLock{
+				nodeID: node1Name,
+			},
+			maxOwners: 2,
+			current: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []lockAnnotationValue{
+					{
+						NodeID:  node2Name,
+						Created: time.Now().UTC().Add(-1 * time.Hour),
+						TTL:     time.Minute,
+					},
+					{
+						NodeID:  node3Name,
+						Created: time.Now().UTC().Add(-1 * time.Minute),
+						TTL:     time.Hour,
+					},
+				},
+			},
+			desired: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []lockAnnotationValue{
+					{NodeID: node1Name},
+					{NodeID: node3Name},
+				},
+			},
+			lockPossible: true,
+		},
+		{
+			name: "full_with_all_expired_locks",
+			daemonSetLock: DaemonSetLock{
+				nodeID: node1Name,
+			},
+			maxOwners: 2,
+			current: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []lockAnnotationValue{
+					{
+						NodeID:  node2Name,
+						Created: time.Now().UTC().Add(-1 * time.Hour),
+						TTL:     time.Minute,
+					},
+					{
+						NodeID:  node3Name,
+						Created: time.Now().UTC().Add(-1 * time.Hour),
+						TTL:     time.Minute,
+					},
+				},
+			},
+			desired: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []lockAnnotationValue{
+					{NodeID: node1Name},
+				},
+			},
+			lockPossible: true,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			lockPossible, actual := testCase.daemonSetLock.canAcquireMultiple(testCase.current, struct{}{}, time.Minute, testCase.maxOwners)
+			if lockPossible != testCase.lockPossible {
+				t.Fatalf(
+					"unexpected result for lock possible (got %t expected %t new annotation %v",
+					lockPossible,
+					testCase.lockPossible,
+					actual,
+				)
+			}
+
+			if lockPossible && (!multiLockAnnotationsAreEqualByNodes(actual, testCase.desired) || testCase.desired.MaxOwners != actual.MaxOwners) {
+				t.Fatalf(
+					"expected lock %v but got %v",
+					testCase.desired,
+					actual,
+				)
+			}
+		})
+	}
+}

pkg/notifications/slack/slack.go

@@ -1,54 +0,0 @@
-package slack
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"time"
-)
-
-var (
-	httpClient = &http.Client{Timeout: 5 * time.Second}
-)
-
-type body struct {
-	Text     string `json:"text,omitempty"`
-	Username string `json:"username,omitempty"`
-	Channel  string `json:"channel,omitempty"`
-}
-
-func notify(hookURL, username, channel, message string) error {
-	msg := body{
-		Text:     message,
-		Username: username,
-		Channel:  channel,
-	}
-
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(&msg); err != nil {
-		return err
-	}
-
-	resp, err := httpClient.Post(hookURL, "application/json", &buf)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf(resp.Status)
-	}
-
-	return nil
-}
-
-// NotifyDrain is the exposed way to notify of a drain event onto a slack chan
-func NotifyDrain(hookURL, username, channel, messageTemplate, nodeID string) error {
-	return notify(hookURL, username, channel, fmt.Sprintf(messageTemplate, nodeID))
-}
-
-// NotifyReboot is the exposed way to notify of a reboot event onto a slack chan
-func NotifyReboot(hookURL, username, channel, messageTemplate, nodeID string) error {
-	return notify(hookURL, username, channel, fmt.Sprintf(messageTemplate, nodeID))
-}

pkg/reboot/command.go

@@ -0,0 +1,25 @@
+package reboot
+
+import (
+	"github.com/kubereboot/kured/pkg/util"
+	log "github.com/sirupsen/logrus"
+)
+
+// CommandRebootMethod holds context-information for a command reboot.
+type CommandRebootMethod struct {
+	nodeID        string
+	rebootCommand []string
+}
+
+// NewCommandReboot creates a new command-rebooter which needs full privileges on the host.
+func NewCommandReboot(nodeID string, rebootCommand []string) *CommandRebootMethod {
+	return &CommandRebootMethod{nodeID: nodeID, rebootCommand: rebootCommand}
+}
+
+// Reboot triggers the command-reboot.
+func (c *CommandRebootMethod) Reboot() {
+	log.Infof("Running command: %s for node: %s", c.rebootCommand, c.nodeID)
+	if err := util.NewCommand(c.rebootCommand[0], c.rebootCommand[1:]...).Run(); err != nil {
+		log.Fatalf("Error invoking reboot command: %v", err)
+	}
+}

pkg/reboot/reboot.go

@@ -0,0 +1,6 @@
+package reboot
+
+// Reboot interface defines the Reboot function to be implemented.
+type Reboot interface {
+	Reboot()
+}

pkg/reboot/signal.go

@@ -0,0 +1,34 @@
+package reboot
+
+import (
+	"os"
+	"syscall"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// SignalRebootMethod holds context-information for a signal reboot.
+type SignalRebootMethod struct {
+	nodeID string
+	signal int
+}
+
+// NewSignalReboot creates a new signal-rebooter which can run unprivileged.
+func NewSignalReboot(nodeID string, signal int) *SignalRebootMethod {
+	return &SignalRebootMethod{nodeID: nodeID, signal: signal}
+}
+
+// Reboot triggers the signal-reboot.
+func (c *SignalRebootMethod) Reboot() {
+	log.Infof("Emit reboot-signal for node: %s", c.nodeID)
+
+	process, err := os.FindProcess(1)
+	if err != nil {
+		log.Fatalf("There was no systemd process found: %v", err)
+	}
+
+	err = process.Signal(syscall.Signal(c.signal))
+	if err != nil {
+		log.Fatalf("Signal of SIGRTMIN+5 failed: %v", err)
+	}
+}

pkg/taints/taints.go

@@ -0,0 +1,166 @@
+package taints
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Taint allows to set soft and hard limitations for scheduling and executing pods on nodes.
+type Taint struct {
+	client    *kubernetes.Clientset
+	nodeID    string
+	taintName string
+	effect    v1.TaintEffect
+	exists    bool
+}
+
+// New provides a new taint.
+func New(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect) *Taint {
+	exists, _, _ := taintExists(client, nodeID, taintName)
+
+	return &Taint{
+		client:    client,
+		nodeID:    nodeID,
+		taintName: taintName,
+		effect:    effect,
+		exists:    exists,
+	}
+}
+
+// Enable creates the taint for a node. Creating an existing taint is a noop.
+func (t *Taint) Enable() {
+	if t.taintName == "" {
+		return
+	}
+
+	if t.exists {
+		return
+	}
+
+	preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, true)
+
+	t.exists = true
+}
+
+// Disable removes the taint for a node. Removing a missing taint is a noop.
+func (t *Taint) Disable() {
+	if t.taintName == "" {
+		return
+	}
+
+	if !t.exists {
+		return
+	}
+
+	preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, false)
+
+	t.exists = false
+}
+
+func taintExists(client *kubernetes.Clientset, nodeID, taintName string) (bool, int, *v1.Node) {
+	updatedNode, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+	if err != nil || updatedNode == nil {
+		log.Fatalf("Error reading node %s: %v", nodeID, err)
+	}
+
+	for i, taint := range updatedNode.Spec.Taints {
+		if taint.Key == taintName {
+			return true, i, updatedNode
+		}
+	}
+
+	return false, 0, updatedNode
+}
+
+func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect, shouldExists bool) {
+	taintExists, offset, updatedNode := taintExists(client, nodeID, taintName)
+
+	if taintExists && shouldExists {
+		log.Debugf("Taint %v exists already for node %v.", taintName, nodeID)
+		return
+	}
+
+	if !taintExists && !shouldExists {
+		log.Debugf("Taint %v already missing for node %v.", taintName, nodeID)
+		return
+	}
+
+	type patchTaints struct {
+		Op    string      `json:"op"`
+		Path  string      `json:"path"`
+		Value interface{} `json:"value,omitempty"`
+	}
+
+	taint := v1.Taint{
+		Key:    taintName,
+		Effect: effect,
+	}
+
+	var patches []patchTaints
+
+	if len(updatedNode.Spec.Taints) == 0 {
+		// add first taint and ensure to keep current taints
+		patches = []patchTaints{
+			{
+				Op:    "test",
+				Path:  "/spec",
+				Value: updatedNode.Spec,
+			},
+			{
+				Op:    "add",
+				Path:  "/spec/taints",
+				Value: []v1.Taint{},
+			},
+			{
+				Op:    "add",
+				Path:  "/spec/taints/-",
+				Value: taint,
+			},
+		}
+	} else if taintExists {
+		// remove taint and ensure to test against race conditions
+		patches = []patchTaints{
+			{
+				Op:    "test",
+				Path:  fmt.Sprintf("/spec/taints/%d", offset),
+				Value: taint,
+			},
+			{
+				Op:   "remove",
+				Path: fmt.Sprintf("/spec/taints/%d", offset),
+			},
+		}
+	} else {
+		// add missing taint to exsting list
+		patches = []patchTaints{
+			{
+				Op:    "add",
+				Path:  "/spec/taints/-",
+				Value: taint,
+			},
+		}
+	}
+
+	patchBytes, err := json.Marshal(patches)
+	if err != nil {
+		log.Fatalf("Error encoding taint patch for node %s: %v", nodeID, err)
+	}
+
+	_, err = client.CoreV1().Nodes().Patch(context.TODO(), nodeID, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
+	if err != nil {
+		log.Fatalf("Error patching taint for node %s: %v", nodeID, err)
+	}
+
+	if shouldExists {
+		log.Info("Node taint added")
+	} else {
+		log.Info("Node taint removed")
+	}
+}

pkg/util/util.go

@@ -0,0 +1,23 @@
+package util
+
+import (
+	"os/exec"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// NewCommand creates a new Command with stdout/stderr wired to our standard logger
+func NewCommand(name string, arg ...string) *exec.Cmd {
+	cmd := exec.Command(name, arg...)
+	cmd.Stdout = log.NewEntry(log.StandardLogger()).
+		WithField("cmd", cmd.Args[0]).
+		WithField("std", "out").
+		WriterLevel(log.InfoLevel)
+
+	cmd.Stderr = log.NewEntry(log.StandardLogger()).
+		WithField("cmd", cmd.Args[0]).
+		WithField("std", "err").
+		WriterLevel(log.WarnLevel)
+
+	return cmd
+}

tests/kind/follow-coordinated-reboot.sh

@@ -46,6 +46,12 @@ do
     echo "${#was_unschedulable[@]} nodes were removed from pool once:" "${!was_unschedulable[@]}"
     echo "${#has_recovered[@]} nodes removed from the pool are now back:" "${!has_recovered[@]}"
 
+    #"$KUBECTL_CMD" logs -n kube-system -l name=kured --ignore-errors > "$tmp_dir"/node_output
+    #if [[ "$DEBUG" == "true" ]]; then
+    #    echo "Kured pod logs:"
+    #    cat "$tmp_dir"/node_output
+    #fi
+
     "$KUBECTL_CMD" get nodes -o custom-columns=NAME:.metadata.name,SCHEDULABLE:.spec.unschedulable --no-headers > "$tmp_dir"/node_output
     if [[ "$DEBUG" == "true" ]]; then
         # This is useful to see if a node gets stuck after drain, and doesn't

tests/kind/test-metrics.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+expected="$1"
+if [[ "$expected" != "0" && "$expected" != "1" ]]; then
+    echo "You should give an argument to this script, the gauge value (0 or 1)"
+    exit 1
+fi
+
+HOST="${HOST:-localhost}"
+PORT="${PORT:-30000}"
+NODENAME="${NODENAME-chart-testing-control-plane}"
+
+reboot_required=$(docker exec "$NODENAME" curl "http://$HOST:$PORT/metrics" | awk '/^kured_reboot_required/{print $2}')
+if [[ "$reboot_required" == "$expected" ]]; then
+    echo "Test success"
+else
+    echo "Test failed"
+    exit 1
+fi