Merge pull request #1248 from kubereboot/dependabot/github_actions/jdx/mise-action-3.4.0

Bumps alpine from 3.22.2 to 3.23.3.

---
updated-dependencies:
- dependency-name: alpine
  dependency-version: 3.23.3
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

.circleci/config.yml

@@ -1,20 +0,0 @@
-version: 2
-jobs:
-  build:
-    working_directory: /go/src/github.com/weaveworks/kured
-    docker:
-      - image: circleci/golang:1.8
-    steps:
-      - checkout
-      - setup_remote_docker
-      - run: go get github.com/golang/dep/cmd/dep
-      - run: dep ensure
-      - run: make
-
-      - deploy:
-          name: Maybe push master images
-          command: |
-            if [ -z "${CIRCLE_TAG}" -a "${CIRCLE_BRANCH}" == "master" ]; then
-              docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" quay.io
-              make publish-image
-            fi

.clomonitor.yml

@@ -0,0 +1,3 @@
+exemptions:
+  - check: analytics
+    reason: "We don't track people"

.github/dependabot.yml

@@ -0,0 +1,21 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  # Maintain dependencies for gomod
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    ignore:
+      - dependency-name: "k8s.io/api"
+      - dependency-name: "k8s.io/apimachinery"
+      - dependency-name: "k8s.io/client-go"
+      - dependency-name: "k8s.io/kubectl"
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/kind-cluster-current.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: "kindest/node:v1.34.0"
+- role: worker
+  image: "kindest/node:v1.34.0"
+- role: worker
+  image: "kindest/node:v1.34.0"

.github/kind-cluster-next.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: "kindest/node:v1.35.0"
+- role: worker
+  image: "kindest/node:v1.35.0"
+- role: worker
+  image: "kindest/node:v1.35.0"

.github/kind-cluster-previous.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: "kindest/node:v1.33.4"
+- role: worker
+  image: "kindest/node:v1.33.4"
+- role: worker
+  image: "kindest/node:v1.33.4"

.github/workflows/codeql.yml

@@ -0,0 +1,83 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "main" ]
+  schedule:
+    - cron: '24 13 * * 6'
+
+permissions:
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+      with:
+        egress-policy: audit
+
+    - name: Checkout repository
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@64d10c13136e1c5bce3e5fbde8d4906eeaafc885 # v3.30.6
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@64d10c13136e1c5bce3e5fbde8d4906eeaafc885 # v3.30.6
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@64d10c13136e1c5bce3e5fbde8d4906eeaafc885 # v3.30.6
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/dependency-review.yml

@@ -0,0 +1,28 @@
+# Dependency Review Action
+#
+# This Action will scan dependency manifest files that change as part of a Pull Request,
+# surfacing known-vulnerable versions of the packages declared or updated in the PR.
+# Once installed, if the workflow run is marked as required,
+# PRs introducing known-vulnerable packages will be blocked from merging.
+#
+# Source repository: https://github.com/actions/dependency-review-action
+name: 'Dependency Review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+        with:
+          egress-policy: audit
+
+      - name: 'Checkout Repository'
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@40c09b7dc99638e5ddb0bfd91c1673effc064d8a # v4.8.1

.github/workflows/on-main-push.yaml

@@ -0,0 +1,89 @@
+# We publish every merged commit in the form of an image
+# named kured:<branch>-<short tag>
+name: Push image of latest main
+on:
+  push:
+    branches:
+      - main
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+permissions:
+  contents: read
+
+jobs:
+  tag-scan-and-push-final-image:
+    name: "Build, scan, and publish tagged image"
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+      packages: write
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Ensure go version
+        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3.5.1
+        with:
+          version: 2025.10.5
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
+
+      - name: Build binaries
+        run: make kured-release-snapshot
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        with:
+          context: .
+          platforms: linux/arm64, linux/amd64, linux/arm/v7, linux/arm/v6, linux/386
+          push: true
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
+
+      - name: Generate SBOM
+        run: |
+          syft ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }} -o spdx > kured.sbom
+
+      - name: Sign and attest artifacts
+        run: |
+          cosign sign -y -r ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
+          cosign sign-blob -y --output-signature kured.sbom.sig --output-certificate kured.sbom.pem kured.sbom
+          cosign attest -y --type spdx --predicate kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
+          cosign attach sbom --type spdx --sbom kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}

.github/workflows/on-pr-docs.yaml

@@ -0,0 +1,26 @@
+name: Verify Docs Links
+on:
+  pull_request:
+  push:
+    paths:
+      - '**.md'
+
+jobs:
+  pr-check-docs-links:
+    name: Check docs for incorrect links
+    runs-on: ubuntu-latest
+    steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+    - name: Link Checker
+      uses: lycheeverse/lychee-action@a8c4c7cb88f0c7386610c35eb25108e448569cb0 # v2.7.0
+      env:
+        GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      with:
+        args: --verbose --no-progress '*.md' '*.yaml' '*/*/*.go' --exclude-link-local
+        fail: true

.github/workflows/on-pr.yaml

@@ -0,0 +1,178 @@
+name: PR
+on:
+  pull_request:
+  push:
+
+jobs:
+  pr-short-tests:
+    name: Run short go tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+        with:
+          egress-policy: audit
+
+      - name: checkout
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Ensure go version
+        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3.5.1
+        with:
+          version: 2025.10.5
+
+      - name: run tests
+        run: make test
+
+      - name: Annotate tests
+        if: always()
+        uses: guyarb/golang-test-annoations@2941118d7ef622b1b3771d1ff6eae9e90659eb26 # v0.8.0
+        with:
+          test-results: test.json
+
+  # This should not be made a mandatory test
+  # It is only used to make us aware of any potential security failure that
+  # should trigger a bump of the image in build/.
+  pr-vuln-scan:
+    name: Build image and scan it against known vulnerabilities
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Ensure go version
+        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3.5.1
+        with:
+          version: 2025.10.5
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
+
+      - name: Build image
+        run: VERSION="${{ steps.tags.outputs.sha_short }}" DH_ORG="${{ github.repository_owner }}" make image
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8
+        with:
+          image-ref: 'ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }}'
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
+
+  # This ensures the latest code works with the manifests built from tree.
+  # It is useful for two things:
+  # - Test manifests changes (obviously), ensuring they don't break existing clusters
+  # - Ensure manifests work with the latest versions even with no manifest change
+  #     (compared to helm charts, manifests cannot easily template changes based on versions)
+  # Helm charts are _trailing_ releases, while manifests are done during development.
+  # This test uses the "command" reboot-method.
+  e2e-manifests:
+    name: End-to-End test with kured with code and manifests from HEAD
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        testname:
+          - "TestE2EWithCommand"
+          - "TestE2EWithSignal"
+          - "TestE2EConcurrentWithCommand"
+          - "TestE2EConcurrentWithSignal"
+        kubernetes_version:
+          - "previous"
+          - "current"
+          - "next"
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Ensure go version
+        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3.5.1
+        with:
+          version: 2025.10.5
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
+
+      - name: Run specific e2e tests
+        run: make e2e-test ARGS="-run ^${{ matrix.testname }}/${{ matrix.kubernetes_version }}"
+
+
+  e2e-tests-singleversion:
+    name: End-to-End test targetting a single version of kubernetes
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        testname:
+          - "TestCordonningIsKept/concurrency1"
+          - "TestCordonningIsKept/concurrency2"
+          - "TestE2EBlocker/podblocker"
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Ensure go version
+        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3.5.1
+        with:
+          version: 2025.10.5
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
+
+      - name: Run specific e2e tests
+        run: make e2e-test ARGS="-run ^${{ matrix.testname }}"

.github/workflows/on-tag.yaml

@@ -0,0 +1,110 @@
+# when we add a tag to the repo, we should publish the kured image to a public repository
+# if it's safe.
+# It doesn't mean it's ready for release, but at least it's getting us started.
+# The next step is to have a PR with the helm chart, to bump the version of the image used
+name: Tag repo
+on:
+  push:
+    tags:
+      - "*"
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+permissions:
+  contents: read
+
+jobs:
+  tag-scan-and-push-final-image:
+    name: "Build, scan, and publish tagged image"
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+      packages: write
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Ensure go version
+        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3.5.1
+        with:
+          version: 2025.10.5
+
+      - name: Find current tag version
+        run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+        id: tags
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Build binaries
+        run: make kured-release-tag
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build single image for scan
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        with:
+          context: .
+          push: false
+          load: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8
+        with:
+          image-ref: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}'
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build release images
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        with:
+          context: .
+          platforms: linux/arm64, linux/amd64, linux/arm/v7, linux/arm/v6, linux/386
+          push: true
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
+
+      - name: Generate SBOM
+        run: |
+          syft ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }} -o spdx > kured.sbom
+
+      - name: Sign and attest artifacts
+        run: |
+          cosign sign -y -r ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
+          cosign sign-blob -y --output-signature kured.sbom.sig kured.sbom
+          cosign attest -y --type spdx --predicate kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
+          cosign attach sbom --type spdx --sbom kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}

.github/workflows/periodics-weekly.yaml

@@ -0,0 +1,116 @@
+name: Daily jobs
+
+on:
+  schedule:
+  - cron: "30 1 * * 6"
+
+jobs:
+  periodics-gotest:
+    name: Run go tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+        with:
+          egress-policy: audit
+
+
+      - name: checkout
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3.5.1
+        with:
+          version: 2025.10.5
+
+      - name: run tests
+        run: make test
+
+      - name: Annotate tests
+        if: always()
+        uses: guyarb/golang-test-annoations@2941118d7ef622b1b3771d1ff6eae9e90659eb26 # v0.8.0
+        with:
+          test-results: test.json
+
+  periodics-mark-stale:
+    name: Mark stale issues and PRs
+    runs-on: ubuntu-latest
+    steps:
+    # Stale, by default, waits for 60 days before marking PR/issues as stale and closes them after 21 days.
+    # Do not expire the first issues that would allow the community to grow.
+    - name: Harden Runner
+      uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+      with:
+        egress-policy: audit
+
+    - uses: actions/stale@5f858e3efba33a5ca4407a664cc011ad407f2008 # v10.1.0
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'This issue was automatically considered stale due to lack of activity. Please update it and/or join our slack channels to promote it, before it automatically closes (in 7 days).'
+        stale-pr-message: 'This PR was automatically considered stale due to lack of activity. Please refresh it and/or join our slack channels to highlight it, before it automatically closes (in 7 days).'
+        stale-issue-label: 'no-issue-activity'
+        stale-pr-label: 'no-pr-activity'
+        exempt-issue-labels: 'good first issue,keep'
+        exempt-pr-labels: 'keep'
+        days-before-close: 21
+
+  check-docs-links:
+    name: Check docs for incorrect links
+    runs-on: ubuntu-latest
+    steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+    - name: Link Checker
+      uses: lycheeverse/lychee-action@a8c4c7cb88f0c7386610c35eb25108e448569cb0
+      env:
+        GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      with:
+        args: --verbose --no-progress '*.md' '*.yaml' '*/*/*.go' --exclude-link-local
+        fail: true
+
+  vuln-scan:
+    name: Build image and scan it against known vulnerabilities
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Ensure go version
+        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3.5.1
+        with:
+          version: 2025.10.5
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Find current tag version
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        id: tags
+
+      - name: Build artifacts
+        run: VERSION="${{ steps.tags.outputs.sha_short }}" DH_ORG="${{ github.repository_owner }}" make image
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8
+        with:
+          image-ref: 'ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }}'
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'

.github/workflows/scorecard.yml

@@ -0,0 +1,78 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '34 3 * * 6'
+  push:
+    branches: [ "main" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
+        with:
+          egress-policy: audit
+
+      - name: "Checkout code"
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v3.pre.node20
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard (optional).
+      # Commenting out will disable upload of results to your repo's Code Scanning dashboard
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@64d10c13136e1c5bce3e5fbde8d4906eeaafc885 # v3.30.6
+        with:
+          sarif_file: results.sarif

.gitignore

@@ -1,4 +1,6 @@
 cmd/kured/kured
-cmd/prom-active-alerts/prom-active-alerts
 vendor
 build
+dist
+test.json
+tests/kind/testfiles/*.yaml

.golangci.yml

@@ -0,0 +1,37 @@
+version: "2"
+#timeout : 5m we can add this if needed 
+modules-download-mode: readonly
+run:
+  tests: false
+linters:
+  enable:
+    - govet
+    - staticcheck
+    - unused
+    - contextcheck
+    - goconst
+    - gosec
+    - testifylint
+    - errcheck
+    - revive
+
+linters-settings:
+  errcheck:
+    check-type-assertions: true
+    check-blank: true
+  revive:
+    severity: warning
+    confidence: 0.8
+    rules:
+      - name: indent-error-flow
+      - name: var-naming
+      - name: import-shadowing
+      # https://github.com/mgechev/revive/blob/HEAD/RULES_DESCRIPTIONS.md#package-comments
+      - name: package-comments # This is not working!
+        disabled: true
+output:
+  format: colored-line-number
+  print-issued-lines: true
+  print-linter-name: true
+  uniq-by-line: false
+  sort-results: true

.goreleaser.yml

@@ -0,0 +1,32 @@
+project_name: kured
+before:
+  hooks:
+    - go mod tidy
+builds:
+  - main: ./cmd/kured
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+      - arm
+      - "386"
+    goarm:
+      - "6"
+      - "7"
+    ldflags:
+      - -s -w -X main.version={{ if .IsSnapshot }}{{ .ShortCommit }}{{ else }}{{ .Version }}{{ end }}
+    mod_timestamp: "{{ .CommitTimestamp }}"
+    flags:
+      - -trimpath
+
+snapshot:
+  name_template: "{{ .ShortCommit }}"
+
+release:
+  disable: true
+
+changelog:
+  skip: true

.lycheeignore

@@ -0,0 +1,6 @@
+app.fossa.com
+cluster.local
+hooks.slack.com
+localhost
+slack://
+teams://

.mise/config.toml

@@ -0,0 +1,8 @@
+[tools]
+cosign = "2.2.3"
+golangci-lint = "2.1.6"
+goreleaser = "1.24.0"
+kind = "0.31.0"
+kubectl = "1.35.0"
+shellcheck = "0.11.0"
+syft = "1.0.1"

CODE_OF_CONDUCT.md

@@ -0,0 +1,3 @@
+# Kured Community Code of Conduct
+
+Kured follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).

CONTRIBUTING.md

@@ -0,0 +1,357 @@
+# Developing `kured`
+
+We love contributions to `kured`, no matter if you are [helping out on
+Slack][slack], reporting or triaging [issues][issues] or contributing code
+to `kured`.
+
+In any case, it will make sense to familiarise yourself with the main
+[documentation][documentation] to understand the different features and
+options, which is helpful for testing. The "building" section in
+particular makes sense if you are planning to contribute code.
+
+[slack]: https://github.com/kubereboot/kured/blob/main/README.md#getting-help
+[issues]: https://github.com/kubereboot/kured/issues
+[documentation]: https://kured.dev/docs
+
+## Prepare your environment
+
+### Your IDE
+
+![JetBrains logo](https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.png)
+
+The core team has access to Goland from [JetBrains][JetBrains]. Huge thanks to them for their sponsorship!
+
+You can use the IDE you prefer. Don't include anything from your IDE in the .gitignore. Please do so in your global .gitignore.
+
+[JetBrains]: https://www.jetbrains.com/
+
+### Basic binaries required
+
+Your system needs at least the following binaries installed:
+
+- make
+- sed
+- find
+- bash (command, echo)
+- docker (for docker buildx)
+- go (see the required version in go.mod)
+- mise (to manage developer tools. See [mise installation instructions](https://mise.jdx.dev/installing-mise.html).)
+
+### Fetch the additional binaries required
+
+Please run `make install-tools` once on a fresh repository clone to download necessary developer tools.
+Installed tools are listed in [.mise directory](.mise/config.toml).
+
+### Configure your git for the "Certificate of Origin"
+
+By contributing to this project, you agree to the Developer Certificate of
+Origin (DCO). This document was created by the Linux Kernel community and is a
+simple statement that you, as a contributor, have the legal right to make the
+contribution.
+
+We require all commits to be signed. By signing off with your signature, you
+certify that you wrote the patch or otherwise have the right to contribute the
+material by the rules of the [DCO](DCO):
+
+`Signed-off-by: Jane Doe <jane.doe@example.com>`
+
+The signature must contain your real name
+(sorry, no pseudonyms or anonymous contributions)
+If your `user.name` and `user.email` are configured in your Git config,
+you can sign your commit automatically with `git commit -s`.
+
+## Get to know Kured repositories
+
+All Kured repositories are kept under <https://github.com/kubereboot>. To find the code and work on the individual pieces that make Kured, here is our overview:
+
+| Repositories                            | Contents                  |
+|-----------------------------------------|---------------------------|
+| <https://github.com/kubereboot/kured>   | Kured operator itself     |
+| <https://github.com/kubereboot/charts>  | Helm chart                |
+| <https://github.com/kubereboot/website> | website and documentation |
+
+We use GitHub Actions in all our repositories.
+
+### Charts repo structure highlights
+
+- We use GitHub Actions to do the chart testing. Only linting/installation happens, no e2e test is done.
+- [charts/kured](https://github.com/kubereboot/charts/tree/main/charts/kured) is the place to contribute changes to the chart. Please bump [Chart.yaml](https://github.com/kubereboot/charts/blob/main/charts/kured/Chart.yaml) at each change according to semver.
+
+### Kured repo structure highlights
+
+Kured's main code can be found in the [`cmd`](cmd) and [`pkg`](pkg) directories
+
+Keep in mind we always want to guarantee that `kured` works for the previous, current, and next
+minor version of kubernetes according to `client-go` and `kubectl` dependencies in use.
+
+Our e2e tests are in the [`tests`](tests) directory. These are deep tests using our manifests with different params, on all supported k8s versions of a release.
+They are expensive but allow us to catch many issues quickly. If you want to ensure your scenario works, add an e2e test for it! Those e2e tests are encouraged by the maintainer team (See below).
+
+We also have other tests:
+
+- golangci-lint , shellcheck
+- a security check against our base image (alpine)
+
+All these tests are run on every PR/tagged release. See [.github/workflows](.github/workflows) for more details.
+
+We use [GoReleaser to build](.goreleaser.yml).
+
+## Regular development activities / maintenance
+
+### Updating k8s support
+
+At each new major release of kubernetes, we update our dependencies.
+
+Beware that whenever we want to update e.g. the `kubectl` or `client-go` dependencies, some other impactful changes might be necessary too.
+(RBAC, drain behaviour changes, ...)
+
+A few examples of what it took to support:
+
+- Kubernetes 1.10 <https://github.com/kubereboot/kured/commit/b3f9ddf> + <https://github.com/kubereboot/kured/commit/bc3f28d> + <https://github.com/kubereboot/kured/commit/908998a> + <https://github.com/kubereboot/kured/commit/efbb0c3> + <https://github.com/kubereboot/kured/commit/5731b98>
+- Kubernetes 1.14 <https://github.com/kubereboot/kured/pull/75>
+- Kubernetes 1.34 <https://github.com/kubereboot/kured/commit/6ab853dd711ee264663184976ae492a20b657b0a>
+
+Search the git log for inspiration for your cases.
+
+In general, the following activities have to happen:
+
+- Bump kind and its images (see below)
+- `go get k8s.io/kubectl@v0.{version}`
+
+### bump kind images support
+
+Go to `.github/workflows` and update the new k8s images. For that:
+
+- `cp .github/kind-cluster-current.yaml .github/kind-cluster-previous.yaml`
+- `cp .github/kind-cluster-next.yaml .github/kind-cluster-current.yaml`
+- Then edit `.github/kind-cluster-next.yaml` to point to the new version.
+
+This will make the full test matrix updated (the CI and the test code).
+
+Once your code passes all tests, update the support matrix in
+the [installation docs](https://kured.dev/docs/installation/).
+
+Beware that sometimes you also need to update the Kind version. grep in the [.github/workflows](.github/workflows) for the kind version.
+
+### Updating other dependencies
+
+Dependabot proposes changes in our `go.mod`/`go.sum`.
+CI testing covers some of those changes, but not all.
+
+Please make sure to test those not covered by CI (mostly the integration
+with other tools) manually before merging.
+
+In all cases, review dependabot changes: Imagine all changes as a possible supply chain
+attack vector. You then need to review the proposed changes by dependabot and evaluate the trust/risks.
+
+### Review periodic jobs
+
+We run periodic jobs (see also Automated testing section of this documentation).
+Those should be monitored for failures.
+
+If a failure happens in periodic testing, something terribly wrong must have happened
+(or GitHub is failing at the creation of a kind cluster). Please monitor those
+failures carefully.
+
+## Testing kured
+
+If you have developed anything (or just want to take kured for a spin!), run the following tests.
+As they will run in CI, we will quickly catch if you did not test before submitting your PR.
+
+### Linting
+
+We use [`golangci-lint`](https://golangci-lint.run/) for Go code linting.
+
+To run lint checks locally:
+
+```bash
+make lint
+```
+
+### Quick Golang code testing
+
+Please run `make test` to run only the basic tests. It gives a good
+idea of the code behaviour.
+
+### Functional testing 
+
+For functional testing, the maintainer team is using `minikube` or `kind` (explained below), but also encourages you to test kured on your own cluster(s).
+
+#### Testing on your own cluster
+
+This will allow the community to catch issues that might not have been tested in our CI, like integration with other tools, or your specific use case.
+
+To test kured on your own cluster, make sure you pass the right `image`, update the `period` and `reboot-days` (so you get immediate results), and update any other flags for your cases.
+Then login to a node and run:
+
+```console
+sudo touch /var/run/reboot-required
+```
+
+Then tell us about everything that went well or went wrong in Slack.
+
+#### Testing with `minikube`
+
+A test-run with `minikube` could look like this:
+
+```cli
+# start minikube
+minikube start --driver=kvm2 --kubernetes-version <k8s-release>
+
+# build kured image and publish to registry accessible by minikube
+make image minikube-publish
+
+# edit kured-ds.yaml to
+#   - point to new image
+#   - change e.g. period and reboot-days option for immediate results
+
+minikube kubectl -- apply -f kured-rbac.yaml
+minikube kubectl -- apply -f kured-ds.yaml
+minikube kubectl -- logs daemonset.apps/kured -n kube-system -f
+
+# In separate terminal
+minikube ssh
+ sudo touch /var/run/reboot-required
+minikube logs -f
+```
+
+Now check for the 'Commanding reboot' message and minikube going down.
+
+Unfortunately, as of today, you are going to run into
+<https://github.com/kubernetes/minikube/issues/2874>. This means that
+minikube won't come back easily. You will need to start minikube again.
+Then you can check for the lock release.
+
+#### Testing with `kind` "The hard way"
+
+A test-run with `kind` could look like this:
+
+```cli
+# create kind cluster
+kind create cluster --config .github/kind-cluster-<k8s-version>.yaml
+
+# create reboot required files on pre-defined kind nodes
+./tests/kind/create-reboot-sentinels.sh
+
+# check if reboot is working fine
+./tests/kind/follow-coordinated-reboot.sh
+
+```
+
+### Testing with `kind` "The easy way"
+
+You can automate the test with `kind` by using the same code as the CI.
+
+```cli
+# Build kured:dev image, build manifests, and run the "long" go tests
+make e2e-test
+```
+
+You can alter the test behaviour by passing arguments to this command.
+A few examples are given below:
+
+```shell
+# Run only TestE2EWithSignal test for the kubernetes version named "current" (see kind file)
+make e2e-test ARGS="-run ^TestE2EWithSignal/current"
+# Run all tests but make sure to extend the timeout, for slower machines.
+make e2e-test ARGS="-timeout 1200s'
+```
+
+## Introducing new features
+
+When you introduce a new feature, the kured team expects you to have tested (see above!)
+your change thoroughly. If possible, include all the necessary testing in your change.
+
+If your change involves a user facing change (change in flags of kured for example),
+please include expose your new feature in our default manifest (`kured-ds.yaml`),
+as a comment.
+
+Our release manifests and helm charts are our stable interfaces.
+Any user facing changes will therefore have to wait for a release before being
+exposed to our users.
+
+This also means that when you expose a new feature, you should create another PR
+for your changes in <https://github.com/kubereboot/charts> to make your feature
+available at the next kured version for helm users.
+
+In the charts PR, you can directly bump the `appVersion` to the next minor version
+(you are introducing a new feature, which requires a bump of the minor number.
+For example, if current `appVersion` is `1.6.x`, make sure you update your `appVersion`
+to `1.7.0`). It allows us to have an easy view of what we land each release.
+
+Do not hesitate to increase the test coverage for your feature, whether it's unit testing 
+to full functional testing (even using helm charts).
+
+The team of kured is small, so we will most likely refuse any feature adding maintenance burden.
+
+## Introducing changes in the helm chart
+
+When you change the helm chart, remember to bump its version according to semver.
+Changes to defaults are frowned upon unless absolutely necessary.
+
+## Introducing new tests / increasing test coverage
+
+On the opposite of features, we welcome ALL features increasing our stability and test coverage.
+See also our GitHub issues with the label [`testing`](https://github.com/kubereboot/kured/labels/testing).
+
+## Publishing a new kured release
+
+### Double-check the latest kubernetes patch version
+
+Ensure you have used the latest patch version in the tree. 
+Check the documentation "Updating k8s support" if the minor version was not yet applied.
+
+### Update the manifests with the new version
+
+```sh
+export VERSION=1.20.0
+make DH_ORG="kubereboot" VERSION="${VERSION}" manifest
+```
+Create a commit updating the manifest with future image [like this one](https://github.com/kubereboot/kured/commit/58091f6145771f426b4b9e012a43a9c847af2560).
+
+### Create the combined manifest for the new release
+
+Now create the `kured-<new version>-combined.yaml` for e.g. `1.20.0`:
+
+```sh
+export VERSION=1.20.0
+export MANIFEST="kured-$VERSION-combined.yaml"
+make DH_ORG="kubereboot" VERSION="${VERSION}" manifest # just to be safe
+cat kured-rbac.yaml > "$MANIFEST"
+cat kured-ds.yaml >> "$MANIFEST"
+```
+
+### Create the new version tag on the repo (optional, can also be done directly in GH web interface)
+
+Tag the previously created commit with the future release version.
+The GitHub Actions workflow will push the new image to the registry.
+
+### Publish new version release artifacts
+
+Now you can head to the GitHub UI for releases, drafting a new
+release. Chose, as tag, the new version number.
+
+Click to generate the release notes.
+
+Fill, as name, "Kured <new version>".
+
+Edit the generated text.
+
+Please describe what's new and noteworthy in the release notes, list the PRs
+that landed and give a shout-out to everyone who contributed.
+Please also note down on which releases the upcoming `kured` release was
+tested on or what it supports. (Check old release notes if you're unsure.)
+
+Before clicking on publishing release, upload the yaml manifest
+(`kured-<new version>-combined.yaml`) file.
+
+Click on publish the release and set as the latest release.
+
+### Prepare Helm chart
+
+Create a commit to [bump the chart and kured version like this one](https://github.com/kubereboot/charts/commit/e0191d91c21db8338be8cbe56f8991a557048110).
+
+### Prepare Documentation
+
+Ensure the [compatibility matrix](https://kured.dev/docs/installation/) is updated to the new version you want to release.
+

DCO

@@ -0,0 +1,36 @@
+Developer Certificate of Origin
+Version 1.1
+
+Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+660 York Street, Suite 102,
+San Francisco, CA 94110 USA
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+
+Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.

DEVELOPMENT.md

@@ -0,0 +1 @@
+This file was moved to [CONTRIBUTING.md](CONTRIBUTING.md).

Dockerfile

@@ -0,0 +1,25 @@
+FROM alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS bin
+
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+COPY dist/ /dist
+RUN set -ex \
+  && case "${TARGETARCH}" in \
+      amd64) \
+          SUFFIX="_v1" \
+          ;; \
+      arm) \
+          SUFFIX="_${TARGETVARIANT:1}" \
+          ;; \
+      *) \
+          SUFFIX="" \
+          ;; \
+    esac \
+  && cp /dist/kured_${TARGETOS}_${TARGETARCH}${SUFFIX}/kured /dist/kured;
+
+FROM alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659
+RUN apk update --no-cache && apk upgrade --no-cache && apk add --no-cache ca-certificates tzdata
+COPY --from=bin /dist/kured /usr/bin/kured
+ENTRYPOINT ["/usr/bin/kured"]

GOVERNANCE.md

@@ -0,0 +1,112 @@
+# Project Governance
+
+- [Values](#values)
+- [Maintainers](#maintainers)
+- [Becoming a Maintainer](#becoming-a-maintainer)
+- [Meetings](#meetings)
+- [Code of Conduct Enforcement](#code-of-conduct)
+- [Voting](#voting)
+
+## Values
+
+The Kured project and its leadership embrace the following values:
+
+- Openness: Communication and decision-making happens in the open and is discoverable for future
+  reference. As much as possible, all discussions and work take place in public
+  forums and open repositories.
+
+- Fairness: All stakeholders have the opportunity to provide feedback and submit
+  contributions, which will be considered on their merits.
+
+- Community over Product or Company: Sustaining and growing our community takes
+  priority over shipping code or sponsors' organizational goals.  Each
+  contributor participates in the project as an individual.
+
+- Inclusivity: We innovate through different perspectives and skill sets, which
+  can only be accomplished in a welcoming and respectful environment.
+
+- Participation: Responsibilities within the project are earned through
+  participation, and there is a clear path up the contributor ladder into leadership
+  positions.
+
+- Consensus: Whether or not wider input is required, the Kured community believes that
+  the best decisions are reached through Consensus
+  <https://en.wikipedia.org/wiki/Consensus_decision-making>.
+
+## Maintainers
+
+Kured Maintainers have write access to the [project GitHub
+organisation](https://github.com/kubereboot). They can merge their own patches or patches
+from others. The current maintainers can be found in [MAINTAINERS][maintainers-file].
+Maintainers collectively manage the project's resources and contributors.
+
+This privilege is granted with some expectation of responsibility: maintainers
+are people who care about the Kured project and want to help it grow and
+improve. A maintainer is not just someone who can make changes, but someone who
+has demonstrated their ability to collaborate with the team, get the most
+knowledgeable people to review code and docs, contribute high-quality code, and
+follow through to fix issues (in code or tests).
+
+A maintainer is a contributor to the project's success and a citizen helping
+the project succeed.
+
+## Becoming a Maintainer
+
+To become a Maintainer you need to demonstrate the following:
+
+- commitment to the project:
+  - participate in discussions, contributions, code and documentation reviews
+      for 3 months or more and participate in Slack discussions and meetings
+      if possible,
+  - perform reviews for 5 non-trivial pull requests,
+  - contribute 5 non-trivial pull requests and have them merged,
+- ability to write quality code and/or documentation,
+- ability to collaborate with the team,
+- understanding of how the team works (policies, processes for testing and code review, etc),
+- understanding of the project's code base and coding and documentation style.
+
+We realise that everybody brings different abilities and qualities to the team, that's
+why we are willing to change the rules somewhat depending on the circumstances.
+
+A new Maintainer can apply by proposing a PR to the [MAINTAINERS
+file][maintainers-file]. A simple majority vote of existing Maintainers
+approves the application.
+
+Maintainers who are selected will be granted the necessary GitHub rights,
+and invited to the [private maintainer mailing list][private-list].
+
+## Meetings
+
+Time zones permitting, Maintainers are expected to participate in the public
+developer meeting, details can be found [here][meeting-agenda].
+
+Maintainers will also have closed meetings in order to discuss security reports
+or Code of Conduct violations.  Such meetings should be scheduled by any
+Maintainer on receipt of a security issue or CoC report.  All current Maintainers
+must be invited to such closed meetings, except for any Maintainer who is
+accused of a CoC violation.
+
+## Code of Conduct
+
+[Code of Conduct](./CODE_OF_CONDUCT.md) violations by community members will
+be discussed and resolved on the [private Maintainer mailing list][private-list].
+If the reported CoC violator is a Maintainer, the Maintainers will instead
+designate two Maintainers to work with CNCF staff in resolving the report.
+
+## Voting
+
+While most business in Kured is conducted by "lazy consensus", periodically
+the Maintainers may need to vote on specific actions or changes.
+A vote can be taken in [kured issues labeled 'decision'][decision-issues] or
+[the private Maintainer mailing list][private-list] for security or conduct
+matters. Votes may also be taken at [the developer meeting][meeting-agenda].
+Any Maintainer may demand a vote be taken.
+
+Most votes require a simple majority of all Maintainers to succeed. Maintainers
+can be removed by a 2/3 majority vote of all Maintainers, and changes to this
+Governance require a 2/3 vote of all Maintainers.
+
+[maintainers-file]: ./MAINTAINERS
+[private-list]: cncf-kured-maintainers@lists.cncf.io
+[meeting-agenda]: https://docs.google.com/document/d/1AWT8YDdqZY-Se6Y1oAlwtujWLVpNVK2M_F_Vfqw06aI/edit
+[decision-issues]: https://github.com/kubereboot/kured/labels/decision

Gopkg.lock

@@ -1,431 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "github.com/PuerkitoBio/purell"
-  packages = ["."]
-  revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/PuerkitoBio/urlesc"
-  packages = ["."]
-  revision = "bbf7a2afc14f93e1e0a5c06df524fbd75e5031e5"
-
-[[projects]]
-  name = "github.com/asaskevich/govalidator"
-  packages = ["."]
-  revision = "73945b6115bfbbcc57d89b7316e28109364124e1"
-  version = "v7"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/beorn7/perks"
-  packages = ["quantile"]
-  revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/cloudfoundry-incubator/candiedyaml"
-  packages = ["."]
-  revision = "99c3df83b51532e3615f851d8c2dbb638f5313bf"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "5215b55f46b2b919f50a1df0eaa5886afe4e3b3d"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/docker/distribution"
-  packages = [
-    "digest",
-    "reference"
-  ]
-  revision = "7365003236ca58bd7fa17ef1459328d13301d7d5"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/emicklei/go-restful"
-  packages = [
-    ".",
-    "log"
-  ]
-  revision = "b14c3a95fc27c52959d2eddc85066da3c14bf269"
-
-[[projects]]
-  name = "github.com/emicklei/go-restful-swagger12"
-  packages = ["."]
-  revision = "dcef7f55730566d41eae5db10e7d6981829720f6"
-  version = "1.0.1"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/ghodss/yaml"
-  packages = ["."]
-  revision = "aa0c862057666179de291b67d9f093d12b5a8473"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/go-openapi/analysis"
-  packages = ["."]
-  revision = "8ed83f2ea9f00f945516462951a288eaa68bf0d6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/go-openapi/errors"
-  packages = ["."]
-  revision = "03cfca65330da08a5a440053faf994a3c682b5bf"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/go-openapi/jsonpointer"
-  packages = ["."]
-  revision = "779f45308c19820f1a69e9a4cd965f496e0da10f"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/go-openapi/jsonreference"
-  packages = ["."]
-  revision = "36d33bfe519efae5632669801b180bf1a245da3b"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/go-openapi/loads"
-  packages = ["."]
-  revision = "a80dea3052f00e5f032e860dd7355cd0cc67e24d"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/go-openapi/spec"
-  packages = ["."]
-  revision = "e51c28f07047ad90caff03f6450908720d337e0c"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/go-openapi/strfmt"
-  packages = ["."]
-  revision = "610b6cacdcde6852f4de68998bd20ce1dac85b22"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/go-openapi/swag"
-  packages = ["."]
-  revision = "24ebf76d720bab64f62824d76bced3184a65490d"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/gogo/protobuf"
-  packages = [
-    "proto",
-    "sortkeys"
-  ]
-  revision = "e33835a643a970c11ac74f6333f5f6866387a101"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/golang/glog"
-  packages = ["."]
-  revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  revision = "2bba0603135d7d7f5cb73b2125beeda19c09f4ef"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/google/gofuzz"
-  packages = ["."]
-  revision = "fd52762d25a41827db7ef64c43756fd4b9f7e382"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/juju/ratelimit"
-  packages = ["."]
-  revision = "5b9ff866471762aa2ab2dced63c9fb6f53921342"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mailru/easyjson"
-  packages = [
-    "buffer",
-    "jlexer",
-    "jwriter"
-  ]
-  revision = "2af9a745a611440bab0528e5ac19b2805a1c50eb"
-
-[[projects]]
-  name = "github.com/matttproud/golang_protobuf_extensions"
-  packages = ["pbutil"]
-  revision = "3247c84500bff8d9fb6d579d800f20b3e091582c"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  revision = "d0303fe809921458f417bcf828397a65db30a7e4"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/client_golang"
-  packages = [
-    "api/prometheus",
-    "prometheus",
-    "prometheus/promhttp"
-  ]
-  revision = "5636dc67ae776adf5590da7349e70fbb9559972d"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/client_model"
-  packages = ["go"]
-  revision = "6f3806018612930941127f2a7c6c453ba2c527d2"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/common"
-  packages = [
-    "expfmt",
-    "internal/bitbucket.org/ww/goautoneg",
-    "model"
-  ]
-  revision = "ebdfc6da46522d58825777cf1f90490a5b1ef1d8"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/procfs"
-  packages = [
-    ".",
-    "xfs"
-  ]
-  revision = "e645f4e5aaa8506fc71d6edbc5c4ff02c04c46f2"
-
-[[projects]]
-  name = "github.com/sirupsen/logrus"
-  packages = ["."]
-  revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc"
-  version = "v1.0.5"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  revision = "b24564e919247d7c870fe0ed3738c98d8741aca4"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  revision = "367864438f1b1a3c7db4da06a2f55b144e6784e0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/ugorji/go"
-  packages = ["codec"]
-  revision = "3487a5545b3d480987dfb0492035299077fab33a"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-  packages = ["ssh/terminal"]
-  revision = "beb2a9779c3b677077c41673505f150149fce895"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "http2",
-    "http2/hpack",
-    "idna"
-  ]
-  revision = "2a35e686583654a1b89ca79c4ac78cb3d6529ca3"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = [
-    "unix",
-    "windows"
-  ]
-  revision = "3b87a42e500a6dc65dae1a55d0b641295971163e"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/text"
-  packages = [
-    "internal/gen",
-    "internal/triegen",
-    "internal/ucd",
-    "transform",
-    "unicode/cldr",
-    "unicode/norm",
-    "width"
-  ]
-  revision = "a9a820217f98f7c8a207ec1e45a874e1fe12c478"
-
-[[projects]]
-  branch = "master"
-  name = "gopkg.in/inf.v0"
-  packages = ["."]
-  revision = "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4"
-
-[[projects]]
-  branch = "v2"
-  name = "gopkg.in/mgo.v2"
-  packages = [
-    "bson",
-    "internal/json"
-  ]
-  revision = "3f83fa5005286a7fe593b055f0d7771a7dce4655"
-
-[[projects]]
-  branch = "v2"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  revision = "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b"
-
-[[projects]]
-  branch = "release-1.7"
-  name = "k8s.io/apimachinery"
-  packages = [
-    "pkg/api/equality",
-    "pkg/api/errors",
-    "pkg/api/meta",
-    "pkg/api/resource",
-    "pkg/apimachinery",
-    "pkg/apimachinery/announced",
-    "pkg/apimachinery/registered",
-    "pkg/apis/meta/v1",
-    "pkg/apis/meta/v1/unstructured",
-    "pkg/apis/meta/v1alpha1",
-    "pkg/conversion",
-    "pkg/conversion/queryparams",
-    "pkg/conversion/unstructured",
-    "pkg/fields",
-    "pkg/labels",
-    "pkg/openapi",
-    "pkg/runtime",
-    "pkg/runtime/schema",
-    "pkg/runtime/serializer",
-    "pkg/runtime/serializer/json",
-    "pkg/runtime/serializer/protobuf",
-    "pkg/runtime/serializer/recognizer",
-    "pkg/runtime/serializer/streaming",
-    "pkg/runtime/serializer/versioning",
-    "pkg/selection",
-    "pkg/types",
-    "pkg/util/clock",
-    "pkg/util/diff",
-    "pkg/util/errors",
-    "pkg/util/framer",
-    "pkg/util/intstr",
-    "pkg/util/json",
-    "pkg/util/net",
-    "pkg/util/rand",
-    "pkg/util/runtime",
-    "pkg/util/sets",
-    "pkg/util/validation",
-    "pkg/util/validation/field",
-    "pkg/util/wait",
-    "pkg/util/yaml",
-    "pkg/version",
-    "pkg/watch",
-    "third_party/forked/golang/reflect"
-  ]
-  revision = "8ab5f3d8a330c2e9baaf84e39042db8d49034ae2"
-
-[[projects]]
-  name = "k8s.io/client-go"
-  packages = [
-    "discovery",
-    "kubernetes",
-    "kubernetes/scheme",
-    "kubernetes/typed/admissionregistration/v1alpha1",
-    "kubernetes/typed/apps/v1beta1",
-    "kubernetes/typed/authentication/v1",
-    "kubernetes/typed/authentication/v1beta1",
-    "kubernetes/typed/authorization/v1",
-    "kubernetes/typed/authorization/v1beta1",
-    "kubernetes/typed/autoscaling/v1",
-    "kubernetes/typed/autoscaling/v2alpha1",
-    "kubernetes/typed/batch/v1",
-    "kubernetes/typed/batch/v2alpha1",
-    "kubernetes/typed/certificates/v1beta1",
-    "kubernetes/typed/core/v1",
-    "kubernetes/typed/extensions/v1beta1",
-    "kubernetes/typed/networking/v1",
-    "kubernetes/typed/policy/v1beta1",
-    "kubernetes/typed/rbac/v1alpha1",
-    "kubernetes/typed/rbac/v1beta1",
-    "kubernetes/typed/settings/v1alpha1",
-    "kubernetes/typed/storage/v1",
-    "kubernetes/typed/storage/v1beta1",
-    "pkg/api",
-    "pkg/api/v1",
-    "pkg/api/v1/ref",
-    "pkg/apis/admissionregistration",
-    "pkg/apis/admissionregistration/v1alpha1",
-    "pkg/apis/apps",
-    "pkg/apis/apps/v1beta1",
-    "pkg/apis/authentication",
-    "pkg/apis/authentication/v1",
-    "pkg/apis/authentication/v1beta1",
-    "pkg/apis/authorization",
-    "pkg/apis/authorization/v1",
-    "pkg/apis/authorization/v1beta1",
-    "pkg/apis/autoscaling",
-    "pkg/apis/autoscaling/v1",
-    "pkg/apis/autoscaling/v2alpha1",
-    "pkg/apis/batch",
-    "pkg/apis/batch/v1",
-    "pkg/apis/batch/v2alpha1",
-    "pkg/apis/certificates",
-    "pkg/apis/certificates/v1beta1",
-    "pkg/apis/extensions",
-    "pkg/apis/extensions/v1beta1",
-    "pkg/apis/networking",
-    "pkg/apis/networking/v1",
-    "pkg/apis/policy",
-    "pkg/apis/policy/v1beta1",
-    "pkg/apis/rbac",
-    "pkg/apis/rbac/v1alpha1",
-    "pkg/apis/rbac/v1beta1",
-    "pkg/apis/settings",
-    "pkg/apis/settings/v1alpha1",
-    "pkg/apis/storage",
-    "pkg/apis/storage/v1",
-    "pkg/apis/storage/v1beta1",
-    "pkg/util",
-    "pkg/util/parsers",
-    "pkg/version",
-    "rest",
-    "rest/watch",
-    "tools/clientcmd/api",
-    "tools/metrics",
-    "transport",
-    "util/cert",
-    "util/flowcontrol",
-    "util/integer"
-  ]
-  revision = "d92e8497f71b7b4e0494e5bd204b48d34bd6f254"
-  version = "v4.0.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "029e6d2251ccbf5acdfc3bc0c36f340dc3a98511b0d7338c3e9bb167e412a155"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,24 +0,0 @@
-
-[[constraint]]
-  name = "github.com/sirupsen/logrus"
-  version = "v1.0.5"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/prometheus/client_golang"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/prometheus/common"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-
-[[constraint]]
-  name = "k8s.io/client-go"
-  version = "v4.0.0"
-
-[[constraint]]
-  name = "k8s.io/apimachinery"
-  branch = "release-1.7"

MAINTAINERS

@@ -0,0 +1,12 @@
+In alphabetical order:
+
+Dharsan Baskar <git@dharsanb.com> (@dharsanb)
+Jack Francis <jackfrancis@gmail.com> (@jackfrancis)
+Jean-Philippe Evrard <open-source@a.spamming.party> (@evrardjp)
+
+Retired maintainers:
+
+- Daniel Holbach
+- Christian Hopf
+
+Thank you for your involvement, and let us not say "farewell" ...

Makefile

@@ -1,34 +1,72 @@
 .DEFAULT: all
-.PHONY: all clean image publish-image minikube-publish
+.PHONY: all clean image minikube-publish manifest test kured-all lint 
 
-DH_ORG=weaveworks
-VERSION=$(shell git symbolic-ref --short HEAD)-$(shell git rev-parse --short HEAD)
+DH_ORG ?= kubereboot
+VERSION=$(shell git rev-parse --short HEAD)
+SUDO=$(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 
 all: image
 
+.PHONY: install-tools
+install-tools:
+	command -v  mise 2>&1 || { echo "please install mise to continue" >&2; exit 127; }
+	mise install
+
 clean:
-	go clean
-	rm -f cmd/kured/kured
-	rm -rf ./build
+	rm -rf ./dist
 
-godeps=$(shell go get $1 && go list -f '{{join .Deps "\n"}}' $1 | grep -v /vendor/ | xargs go list -f '{{if not .Standard}}{{ $$dep := . }}{{range .GoFiles}}{{$$dep.Dir}}/{{.}} {{end}}{{end}}')
+kured:
+	goreleaser build --clean --single-target --snapshot
 
-DEPS=$(call godeps,./cmd/kured)
+kured-all:
+	goreleaser build --clean --snapshot
 
-cmd/kured/kured: $(DEPS)
-cmd/kured/kured: cmd/kured/*.go
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X main.version=$(VERSION)" -o $@ cmd/kured/*.go
+kured-release-tag:
+	goreleaser release --clean
 
-build/.image.done: cmd/kured/Dockerfile cmd/kured/kured
-	mkdir -p build
-	cp $^ build
-	sudo -E docker build -t quay.io/$(DH_ORG)/kured:$(VERSION) -f build/Dockerfile ./build
-	touch $@
+kured-release-snapshot:
+	goreleaser release --clean --snapshot
 
-image: build/.image.done
+image: kured
+	$(SUDO) docker buildx build --no-cache --load -t ghcr.io/$(DH_ORG)/kured:$(VERSION) .
 
-publish-image: image
-	sudo -E docker push quay.io/$(DH_ORG)/kured:$(VERSION)
+dev-image: image
+	$(SUDO) docker tag ghcr.io/$(DH_ORG)/kured:$(VERSION) kured:dev
+
+dev-manifest:
+	# basic e2e scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' kured-ds.yaml > tests/kind/testfiles/kured-ds.yaml
+	# signal e2e scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' kured-ds-signal.yaml > tests/kind/testfiles/kured-ds-signal.yaml
+	# concurrency e2e command scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' -e 's/#\(.*\)--concurrency=1/\1--concurrency=2/g' kured-ds.yaml > tests/kind/testfiles/kured-ds-concurrent-command.yaml
+	# concurrency e2e signal scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' -e 's/#\(.*\)--concurrency=1/\1--concurrency=2/g' kured-ds-signal.yaml > tests/kind/testfiles/kured-ds-concurrent-signal.yaml
+	# pod blocker e2e signal scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' -e 's/#\(.*\)--blocking-pod-selector=name=temperamental/\1--blocking-pod-selector=app=blocker/g' kured-ds-signal.yaml > tests/kind/testfiles/kured-ds-podblocker.yaml
+
+e2e-test: dev-manifest dev-image
+	echo "Running ALL go tests"
+	go test -count=1 -v --parallel 4 ./... $(ARGS)
 
 minikube-publish: image
-	sudo -E docker save quay.io/$(DH_ORG)/kured:$(VERSION) | (eval $$(minikube docker-env) && docker load)
+	$(SUDO) docker save ghcr.io/$(DH_ORG)/kured | (eval $$(minikube docker-env) && docker load)
+
+manifest:
+	sed -i "s#image: ghcr.io/.*kured.*#image: ghcr.io/$(DH_ORG)/kured:$(VERSION)#g" kured-ds.yaml
+	sed -i "s#image: ghcr.io/.*kured.*#image: ghcr.io/$(DH_ORG)/kured:$(VERSION)#g" kured-ds-signal.yaml
+	echo "Please generate combined manifest if necessary"
+
+test: lint
+	@echo "Running short go tests"
+	go test -test.short -json ./... > test.json
+
+lint:
+	@echo "Running shellcheck"
+	find . -name '*.sh' | xargs -n1 shellcheck
+	@echo "Running golangci-lint..."
+	golangci-lint run ./...
+
+lint-docs:
+	@echo "Running lychee"
+	mise x lychee@latest -- lychee --verbose --no-progress '*.md' '*.yaml' '*/*/*.go' --exclude-link-local

README.md

@@ -1,21 +1,18 @@
+# kured - Kubernetes Reboot Daemon
 
-<img src="https://github.com/weaveworks/kured/raw/master/img/logo.png" align="right"/>
+[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/kured)](https://artifacthub.io/packages/helm/kured/kured)
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkubereboot%2Fkured.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkubereboot%2Fkured?ref=badge_shield)
+[![CLOMonitor](https://img.shields.io/endpoint?url=https://clomonitor.io/api/projects/cncf/kured/badge)](https://clomonitor.io/projects/cncf/kured)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8867/badge)](https://www.bestpractices.dev/projects/8867)
 
-* [Introduction](#introduction)
-* [Kubernetes & OS Compatibility](#kubernetes-&-os-compatibility)
-* [Installation](#installation)
-* [Configuration](#configuration)
-	* [Reboot Sentinel File & Period](#reboot-sentinel-file-&-period)
-	* [Blocking Reboots via Alerts](#blocking-reboots-via-alerts)
-	* [Prometheus Metrics](#prometheus-metrics)
-	* [Slack Notifications](#slack-notifications)
-	* [Overriding Lock Configuration](#overriding-lock-configuration)
-* [Operation](#operation)
-	* [Testing](#testing)
-	* [Disabling Reboots](#disabling-reboots)
-	* [Manual Unlock](#manual-unlock)
-* [Building](#building)
-* [Getting Help](#getting-help)
+<img src="https://github.com/kubereboot/website/raw/main/static/img/kured.png" alt="kured logo" width="200" align="right"/>
+
+- [kured - Kubernetes Reboot Daemon](#kured---kubernetes-reboot-daemon)
+  - [Introduction](#introduction)
+  - [Documentation](#documentation)
+  - [Getting Help](#getting-help)
+  - [Trademarks](#trademarks)
+  - [License](#license)
 
 ## Introduction
 
@@ -23,188 +20,48 @@ Kured (KUbernetes REboot Daemon) is a Kubernetes daemonset that
 performs safe automatic node reboots when the need to do so is
 indicated by the package management system of the underlying OS.
 
-* Watches for the presence of a reboot sentinel e.g. `/var/run/reboot-required`
-* Utilises a lock in the API server to ensure only one node reboots at
+- Watches for the presence of a reboot sentinel file e.g. `/var/run/reboot-required`
+  or the successful run of a sentinel command.
+- Utilises a lock in the API server to ensure only one node reboots at
   a time
-* Optionally defers reboots in the presence of active Prometheus alerts
-* Cordons & drains worker nodes before reboot, uncordoning them after
+- Optionally defers reboots in the presence of active Prometheus alerts or selected pods
+- Cordons & drains worker nodes before reboot, uncordoning them after
 
-## Kubernetes & OS Compatibility
+## Documentation
 
-The daemon image contains a 1.7.x `k8s.io/client-go` and `kubectl`
-binary for the purposes of maintaining the lock and draining worker
-nodes. Whilst it has only been tested on a 1.7.x cluster, Kubernetes
-typically has good forwards/backwards compatibility so there is a
-reasonable chance it will work on adjacent versions; please file an
-issue if this is not the case.
+Find all our docs on <https://kured.dev>:
 
-Additionally, the image contains a `systemctl` binary from Ubuntu
-16.04 in order to command reboots. Again, although this has not been
-tested against other systemd distributions there is a good chance that
-it will work.
+- [All Kured Documentation](https://kured.dev/docs/)
+- [Installing Kured](https://kured.dev/docs/installation/)
+- [Configuring Kured](https://kured.dev/docs/configuration/)
+- [Operating Kured](https://kured.dev/docs/operation/)
+- [Developing Kured](https://kured.dev/docs/development/)
 
-## Installation
-
-To obtain a default installation without Prometheus alerting interlock
-or Slack notifications:
-
-```
-kubectl apply -f https://github.com/weaveworks/kured/releases/download/1.0.0/kured-ds.yaml
-```
-
-If you want to customise the installation, download the manifest and
-edit it in accordance with the following section before application.
-
-## Configuration
-
-The following arguments can be passed to kured via the daemonset pod template:
-
-```
-Flags:
-      --alert-filter-regexp value   alert names to ignore when checking for active alerts
-      --ds-name string              namespace containing daemonset on which to place lock (default "kube-system")
-      --ds-namespace string         name of daemonset on which to place lock (default "kured")
-      --lock-annotation string      annotation in which to record locking node (default "weave.works/kured-node-lock")
-      --period duration             reboot check period (default 1h0m0s)
-      --prometheus-url string       Prometheus instance to probe for active alerts
-      --reboot-sentinel string      path to file whose existence signals need to reboot (default "/var/run/reboot-required")
-      --slack-hook-url string       slack hook URL for reboot notfications
-      --slack-username string       slack username for reboot notfications (default "kured")
-```
-
-### Reboot Sentinel File & Period
-
-By default kured checks for the existence of
-`/var/run/reboot-required` every sixty minutes; you can override these
-values with `--reboot-sentinel` and `--period`. Each replica of the
-daemon uses a random offset derived from the period on startup so that
-nodes don't all contend for the lock simultaneously.
-
-### Blocking Reboots via Alerts
-
-You may find it desirable to block automatic node reboots when there
-are active alerts - you can do so by providing the URL of your
-Prometheus server:
-
-```
---prometheus-url=http://prometheus.monitoring.svc.cluster.local
-```
-
-By default the presence of *any* active (pending or firing) alerts
-will block reboots, however you can ignore specific alerts:
-
-```
---alert-filter-regexp=^(RebootRequired|AnotherBenignAlert|...$
-```
-
-An important application of this filter will become apparent in the
-next section.
-
-### Prometheus Metrics
-
-Each kured pod exposes a single gauge metric (`:8080/metrics`) that
-indicates the presence of the sentinel file:
-
-```
-# HELP kured_reboot_required OS requires reboot due to software updates.
-# TYPE kured_reboot_required gauge
-kured_reboot_required{node="ip-xxx-xxx-xxx-xxx.ec2.internal"} 0
-```
-
-The purpose of this metric is to power an alert which will summon an
-operator if the cluster cannot reboot itself automatically for a
-prolonged period:
-
-```
-# Alert if a reboot is required for any machines. Acts as a failsafe for the
-# reboot daemon, which will not reboot nodes if there are pending alerts save
-# this one.
-ALERT RebootRequired
-  IF          max(kured_reboot_required) != 0
-  FOR         24h
-  LABELS      { severity="warning" }
-  ANNOTATIONS {
-    summary = "Machine(s) require being rebooted, and the reboot daemon has failed to do so for 24 hours",
-    impact = "Cluster nodes more vulnerable to security exploits. Eventually, no disk space left.",
-    description = "Machine(s) require being rebooted, probably due to kernel update.",
-  }
-```
-
-If you choose to employ such an alert and have configured kured to
-probe for active alerts before rebooting, be sure to specify
-`--alert-filter-regexp=^RebootRequired$` to avoid deadlock!
-
-### Slack Notifications
-
-If you specify a Slack hook via `--slack-hook-url`, kured will notify
-you immediately prior to rebooting a node:
-
-<img src="https://github.com/weaveworks/kured/raw/master/img/slack-notification.png"/>
-
-We recommend setting `--slack-username` to be the name of the
-environment, e.g. `dev` or `prod`.
-
-### Overriding Lock Configuration
-
-The `--ds-name` and `--ds-namespace` arguments should match the name and
-namespace of the daemonset used to deploy the reboot daemon - the locking is
-implemented by means of an annotation on this resource. The defaults match
-the daemonset YAML provided in the repository.
-
-Similarly `--lock-annotation` can be used to change the name of the
-annotation kured will use to store the lock, but the default is almost
-certainly safe.
-
-## Operation
-
-The example commands in this section assume that you have not
-overriden the default lock annotation, daemonset name or namespace;
-if you have, you will have to adjust the commands accordingly.
-
-### Testing
-
-You can test your configuration by provoking a reboot on a node:
-
-```
-sudo touch /var/run/reboot-required
-```
-
-### Disabling Reboots
-
-If you need to temporarily stop kured from rebooting any nodes, you
-can take the lock manually:
-
-```
-kubectl -n kube-system annotate ds kured weave.works/kured-node-lock='{"nodeID":"manual"}'
-```
-
-Don't forget to release it afterwards!
-
-### Manual Unlock
-
-In exceptional circumstances, such as a node experiencing a permanent
-failure whilst rebooting, manual intervention may be required to
-remove the cluster lock:
-
-```
-kubectl -n kube-system annotate ds kured weave.works/kured-node-lock-
-```
-> NB the `-` at the end of the command is important - it instructs
-> `kubectl` to remove that annotation entirely.
-
-## Building
-
-```
-dep ensure && make
-```
+And there's much more!
 
 ## Getting Help
 
 If you have any questions about, feedback for or problems with `kured`:
 
-- Invite yourself to the <a href="https://weaveworks.github.io/community-slack/" target="_blank"> #weave-community </a> slack channel.
-- Ask a question on the <a href="https://weave-community.slack.com/messages/general/"> #weave-community</a> slack channel.
-- Send an email to <a href="mailto:weave-users@weave.works">weave-users@weave.works</a>
-- <a href="https://github.com/weaveworks/kured/issues/new">File an issue.</a>
+- Invite yourself to the <a href="https://slack.cncf.io/" target="_blank">CNCF Slack</a>.
+- Ask a question on the [#kured](https://cloud-native.slack.com/archives/kured) slack channel.
+- [File an issue](https://github.com/kubereboot/kured/issues/new).
+- Join us in [our monthly meeting](https://docs.google.com/document/d/1AWT8YDdqZY-Se6Y1oAlwtujWLVpNVK2M_F_Vfqw06aI/edit),
+  every first Wednesday of the month at 16:00 UTC.
+- You might want to [join the kured-dev mailing list](https://lists.cncf.io/g/cncf-kured-dev) as well.
+
+We follow the [CNCF Code of Conduct](CODE_OF_CONDUCT.md).
 
 Your feedback is always welcome!
+
+## Trademarks
+
+**Kured is a [Cloud Native Computing Foundation](https://cncf.io/) Sandbox project.**
+
+![Cloud Native Computing Foundation logo](img/cncf-color.png)
+
+The Linux Foundation® (TLF) has registered trademarks and uses trademarks. For a list of TLF trademarks, see [Trademark Usage](https://www.linuxfoundation.org/legal/trademark-usage).
+
+## License
+
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkubereboot%2Fkured.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkubereboot%2Fkured?ref=badge_large)

cmd/kured/Dockerfile

@@ -1,6 +0,0 @@
-FROM ubuntu:16.04
-RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/cache/apt
-ADD https://storage.googleapis.com/kubernetes-release/release/v1.9.6/bin/linux/amd64/kubectl /usr/bin/kubectl
-RUN chmod 0755 /usr/bin/kubectl
-COPY ./kured /usr/bin/kured
-ENTRYPOINT ["/usr/bin/kured"]

cmd/kured/main.go

@@ -1,40 +1,92 @@
+// The main controller for kured
+// This package is a reference implementation on how to reboot your nodes based on the different
+// tools present in this project's modules
 package main
 
 import (
+	"context"
+	"encoding/json"
+	"fmt"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"os"
-	"os/exec"
-	"regexp"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
 	"time"
 
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-
+	"github.com/containrrr/shoutrrr"
+	"github.com/kubereboot/kured/internal"
+	"github.com/kubereboot/kured/pkg/blockers"
+	"github.com/kubereboot/kured/pkg/checkers"
+	"github.com/kubereboot/kured/pkg/daemonsetlock"
+	"github.com/kubereboot/kured/pkg/delaytick"
+	"github.com/kubereboot/kured/pkg/reboot"
+	"github.com/kubereboot/kured/pkg/taints"
+	"github.com/kubereboot/kured/pkg/timewindow"
+	papi "github.com/prometheus/client_golang/api"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/weaveworks/kured/pkg/alerts"
-	"github.com/weaveworks/kured/pkg/daemonsetlock"
-	"github.com/weaveworks/kured/pkg/delaytick"
-	"github.com/weaveworks/kured/pkg/notifications/slack"
+	log "github.com/sirupsen/logrus"
+	flag "github.com/spf13/pflag"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	kubectldrain "k8s.io/kubectl/pkg/drain"
 )
 
 var (
 	version = "unreleased"
 
 	// Command line flags
-	period         time.Duration
-	dsNamespace    string
-	dsName         string
-	lockAnnotation string
-	prometheusURL  string
-	alertFilter    *regexp.Regexp
-	rebootSentinel string
-	slackHookURL   string
-	slackUsername  string
+	forceReboot                     bool
+	drainDelay                      time.Duration
+	drainTimeout                    time.Duration
+	rebootDelay                     time.Duration
+	rebootMethod                    string
+	period                          time.Duration
+	metricsHost                     string
+	metricsPort                     int
+	drainGracePeriod                int
+	drainPodSelector                string
+	skipWaitForDeleteTimeoutSeconds int
+	dsNamespace                     string
+	dsName                          string
+	lockAnnotation                  string
+	lockTTL                         time.Duration
+	lockReleaseDelay                time.Duration
+	prometheusURL                   string
+	preferNoScheduleTaintName       string
+	alertFilter                     regexpValue
+	alertFilterMatchOnly            bool
+	alertFiringOnly                 bool
+	rebootSentinelFile              string
+	rebootSentinelCommand           string
+	notifyURL                       string
+	slackHookURL                    string
+	slackUsername                   string
+	slackChannel                    string
+	messageTemplateDrain            string
+	messageTemplateReboot           string
+	messageTemplateUncordon         string
+	podSelectors                    []string
+	rebootCommand                   string
+	rebootSignal                    int
+	logFormat                       string
+	preRebootNodeLabels             []string
+	postRebootNodeLabels            []string
+	nodeID                          string
+	concurrency                     int
+
+	rebootDays    []string
+	rebootStart   string
+	rebootEnd     string
+	timezone      string
+	annotateNodes bool
 
 	// Metrics
 	rebootRequiredGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -44,184 +96,165 @@ var (
 	}, []string{"node"})
 )
 
+const (
+	// KuredNodeLockAnnotation is the canonical string value for the kured node-lock annotation
+	KuredNodeLockAnnotation string = "weave.works/kured-node-lock"
+	// KuredRebootInProgressAnnotation is the canonical string value for the kured reboot-in-progress annotation
+	KuredRebootInProgressAnnotation string = "weave.works/kured-reboot-in-progress"
+	// KuredMostRecentRebootNeededAnnotation is the canonical string value for the kured most-recent-reboot-needed annotation
+	KuredMostRecentRebootNeededAnnotation string = "weave.works/kured-most-recent-reboot-needed"
+	// EnvPrefix The environment variable prefix of all environment variables bound to our command line flags.
+	EnvPrefix = "KURED"
+
+	sigTrminPlus5 = 34 + 5
+)
+
 func init() {
 	prometheus.MustRegister(rebootRequiredGauge)
 }
 
 func main() {
-	rootCmd := &cobra.Command{
-		Use:   "kured",
-		Short: "Kubernetes Reboot Daemon",
-		Run:   root}
 
-	rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60,
-		"reboot check period")
-	rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system",
+	flag.StringVar(&nodeID, "node-id", "",
+		"node name kured runs on, should be passed down from spec.nodeName via KURED_NODE_ID environment variable")
+	flag.BoolVar(&forceReboot, "force-reboot", false,
+		"force a reboot even if the drain fails or times out")
+	flag.StringVar(&metricsHost, "metrics-host", "",
+		"host where metrics will listen")
+	flag.IntVar(&metricsPort, "metrics-port", 8080,
+		"port number where metrics will listen")
+	flag.IntVar(&drainGracePeriod, "drain-grace-period", -1,
+		"time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used")
+	flag.StringVar(&drainPodSelector, "drain-pod-selector", "",
+		"only drain pods with labels matching the selector (default: '', all pods)")
+	flag.IntVar(&skipWaitForDeleteTimeoutSeconds, "skip-wait-for-delete-timeout", 0,
+		"when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node")
+	flag.DurationVar(&drainDelay, "drain-delay", 0,
+		"delay drain for this duration (default: 0, disabled)")
+	flag.DurationVar(&drainTimeout, "drain-timeout", 0,
+		"timeout after which the drain is aborted (default: 0, infinite time)")
+	flag.DurationVar(&rebootDelay, "reboot-delay", 0,
+		"delay reboot for this duration (default: 0, disabled)")
+	flag.StringVar(&rebootMethod, "reboot-method", "command",
+		"method to use for reboots. Available: command")
+	flag.DurationVar(&period, "period", time.Minute*60,
+		"sentinel check period")
+	flag.StringVar(&dsNamespace, "ds-namespace", "kube-system",
 		"namespace containing daemonset on which to place lock")
-	rootCmd.PersistentFlags().StringVar(&dsName, "ds-name", "kured",
+	flag.StringVar(&dsName, "ds-name", "kured",
 		"name of daemonset on which to place lock")
-	rootCmd.PersistentFlags().StringVar(&lockAnnotation, "lock-annotation", "weave.works/kured-node-lock",
+	flag.StringVar(&lockAnnotation, "lock-annotation", KuredNodeLockAnnotation,
 		"annotation in which to record locking node")
-	rootCmd.PersistentFlags().StringVar(&prometheusURL, "prometheus-url", "",
+	flag.DurationVar(&lockTTL, "lock-ttl", 0,
+		"expire lock annotation after this duration (default: 0, disabled)")
+	flag.DurationVar(&lockReleaseDelay, "lock-release-delay", 0,
+		"delay lock release for this duration (default: 0, disabled)")
+	flag.StringVar(&prometheusURL, "prometheus-url", "",
 		"Prometheus instance to probe for active alerts")
-	rootCmd.PersistentFlags().Var(&regexpValue{&alertFilter}, "alert-filter-regexp",
+	flag.Var(&alertFilter, "alert-filter-regexp",
 		"alert names to ignore when checking for active alerts")
-	rootCmd.PersistentFlags().StringVar(&rebootSentinel, "reboot-sentinel", "/var/run/reboot-required",
-		"path to file whose existence signals need to reboot")
+	flag.BoolVar(&alertFilterMatchOnly, "alert-filter-match-only", false,
+		"Only block if the alert-filter-regexp matches active alerts")
+	flag.BoolVar(&alertFiringOnly, "alert-firing-only", false,
+		"only consider firing alerts when checking for active alerts")
+	flag.StringVar(&rebootSentinelFile, "reboot-sentinel", "/var/run/reboot-required",
+		"path to file whose existence triggers the reboot command")
+	flag.StringVar(&preferNoScheduleTaintName, "prefer-no-schedule-taint", "",
+		"Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Disabled by default. Set e.g. to \"weave.works/kured-node-reboot\" to enable tainting.")
+	flag.StringVar(&rebootSentinelCommand, "reboot-sentinel-command", "",
+		"command for which a zero return code will trigger a reboot command")
+	flag.StringVar(&rebootCommand, "reboot-command", "/bin/systemctl reboot",
+		"command to run when a reboot is required")
+	flag.IntVar(&concurrency, "concurrency", 1,
+		"amount of nodes to concurrently reboot. Defaults to 1")
+	flag.IntVar(&rebootSignal, "reboot-signal", sigTrminPlus5,
+		"signal to use for reboot, SIGRTMIN+5 by default.")
+	flag.StringVar(&slackHookURL, "slack-hook-url", "",
+		"slack hook URL for reboot notifications [deprecated in favor of --notify-url]")
+	flag.StringVar(&slackUsername, "slack-username", "kured",
+		"slack username for reboot notifications")
+	flag.StringVar(&slackChannel, "slack-channel", "",
+		"slack channel for reboot notifications")
+	flag.StringVar(&notifyURL, "notify-url", "",
+		"notify URL for reboot notifications (cannot use with --slack-hook-url flags)")
+	flag.StringVar(&messageTemplateUncordon, "message-template-uncordon", "Node %s rebooted & uncordoned successfully!",
+		"message template used to notify about a node being successfully uncordoned")
+	flag.StringVar(&messageTemplateDrain, "message-template-drain", "Draining node %s",
+		"message template used to notify about a node being drained")
+	flag.StringVar(&messageTemplateReboot, "message-template-reboot", "Rebooting node %s",
+		"message template used to notify about a node being rebooted")
+	flag.StringArrayVar(&podSelectors, "blocking-pod-selector", nil,
+		"label selector identifying pods whose presence should prevent reboots")
+	flag.StringSliceVar(&rebootDays, "reboot-days", timewindow.EveryDay,
+		"schedule reboot on these days")
+	flag.StringVar(&rebootStart, "start-time", "0:00",
+		"schedule reboot only after this time of day")
+	flag.StringVar(&rebootEnd, "end-time", "23:59:59",
+		"schedule reboot only before this time of day")
+	flag.StringVar(&timezone, "time-zone", "UTC",
+		"use this timezone for schedule inputs")
+	flag.BoolVar(&annotateNodes, "annotate-nodes", false,
+		"if set, the annotations 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' will be given to nodes undergoing kured reboots")
+	flag.StringVar(&logFormat, "log-format", "text",
+		"use text or json log format")
+	flag.StringSliceVar(&preRebootNodeLabels, "pre-reboot-node-labels", nil,
+		"labels to add to nodes before cordoning")
+	flag.StringSliceVar(&postRebootNodeLabels, "post-reboot-node-labels", nil,
+		"labels to add to nodes after uncordoning")
 
-	rootCmd.PersistentFlags().StringVar(&slackHookURL, "slack-hook-url", "",
-		"slack hook URL for reboot notfications")
-	rootCmd.PersistentFlags().StringVar(&slackUsername, "slack-username", "kured",
-		"slack username for reboot notfications")
+	flag.Parse()
 
-	if err := rootCmd.Execute(); err != nil {
-		log.Fatal(err)
+	// Load flags from environment variables
+	LoadFromEnv()
+
+	log.Infof("Kubernetes Reboot Daemon: %s", version)
+
+	if logFormat == "json" {
+		log.SetFormatter(&log.JSONFormatter{})
 	}
-}
 
-// newCommand creates a new Command with stdout/stderr wired to our standard logger
-func newCommand(name string, arg ...string) *exec.Cmd {
-	cmd := exec.Command(name, arg...)
-
-	cmd.Stdout = log.NewEntry(log.StandardLogger()).
-		WithField("cmd", cmd.Args[0]).
-		WithField("std", "out").
-		WriterLevel(log.InfoLevel)
-
-	cmd.Stderr = log.NewEntry(log.StandardLogger()).
-		WithField("cmd", cmd.Args[0]).
-		WithField("std", "err").
-		WriterLevel(log.WarnLevel)
-
-	return cmd
-}
-
-func sentinelExists() bool {
-	_, err := os.Stat(rebootSentinel)
-	switch {
-	case err == nil:
-		return true
-	case os.IsNotExist(err):
-		return false
-	default:
-		log.Fatalf("Unable to determine existence of sentinel: %v", err)
-		return false // unreachable; prevents compilation error
+	if nodeID == "" {
+		log.Fatal("KURED_NODE_ID environment variable required")
 	}
-}
+	log.Infof("Node ID: %s", nodeID)
 
-func rebootRequired() bool {
-	if sentinelExists() {
-		log.Infof("Reboot required")
-		return true
-	} else {
-		log.Infof("Reboot not required")
-		return false
-	}
-}
+	notifyURL = validateNotificationURL(notifyURL, slackHookURL)
 
-func rebootBlocked() bool {
-	if prometheusURL != "" {
-		alertNames, err := alerts.PrometheusActiveAlerts(prometheusURL, alertFilter)
-		if err != nil {
-			log.Warnf("Reboot blocked: prometheus query error: %v", err)
-			return true
-		}
-		count := len(alertNames)
-		if count > 10 {
-			alertNames = append(alertNames[:10], "...")
-		}
-		if count > 0 {
-			log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
-			return true
-		}
-	}
-	return false
-}
-
-func holding(lock *daemonsetlock.DaemonSetLock, metadata interface{}) bool {
-	holding, err := lock.Test(metadata)
+	err := validateNodeLabels(preRebootNodeLabels, postRebootNodeLabels)
 	if err != nil {
-		log.Fatalf("Error testing lock: %v", err)
-	}
-	if holding {
-		log.Infof("Holding lock")
-	}
-	return holding
-}
-
-func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}) bool {
-	holding, holder, err := lock.Acquire(metadata)
-	switch {
-	case err != nil:
-		log.Fatalf("Error acquiring lock: %v", err)
-		return false
-	case !holding:
-		log.Warnf("Lock already held: %v", holder)
-		return false
-	default:
-		log.Infof("Acquired reboot lock")
-		return true
-	}
-}
-
-func release(lock *daemonsetlock.DaemonSetLock) {
-	log.Infof("Releasing lock")
-	if err := lock.Release(); err != nil {
-		log.Fatalf("Error releasing lock: %v", err)
-	}
-}
-
-func drain(nodeID string) {
-	log.Infof("Draining node %s", nodeID)
-	drainCmd := newCommand("/usr/bin/kubectl", "drain",
-		"--ignore-daemonsets", "--delete-local-data", "--force", nodeID)
-
-	if err := drainCmd.Run(); err != nil {
-		log.Fatalf("Error invoking drain command: %v", err)
-	}
-}
-
-func uncordon(nodeID string) {
-	log.Infof("Uncordoning node %s", nodeID)
-	uncordonCmd := newCommand("/usr/bin/kubectl", "uncordon", nodeID)
-	if err := uncordonCmd.Run(); err != nil {
-		log.Fatalf("Error invoking uncordon command: %v", err)
-	}
-}
-
-func commandReboot(nodeID string) {
-	log.Infof("Commanding reboot")
-
-	if slackHookURL != "" {
-		if err := slack.NotifyReboot(slackHookURL, slackUsername, nodeID); err != nil {
-			log.Warnf("Error notifying slack: %v", err)
-		}
+		log.Warn(err.Error())
 	}
 
-	// Relies on /var/run/dbus/system_bus_socket bind mount to talk to systemd
-	rebootCmd := newCommand("/bin/systemctl", "reboot")
-	if err := rebootCmd.Run(); err != nil {
-		log.Fatalf("Error invoking reboot command: %v", err)
+	log.Infof("PreferNoSchedule taint: %s", preferNoScheduleTaintName)
+
+	// This should be printed from blocker list instead of only blocking pod selectors
+	log.Infof("Blocking Pod Selectors: %v", podSelectors)
+
+	log.Infof("Reboot period %v", period)
+	log.Infof("Concurrency: %v", concurrency)
+
+	if annotateNodes {
+		log.Infof("Will annotate nodes during kured reboot operations")
 	}
-}
 
-func maintainRebootRequiredMetric(nodeID string) {
-	for {
-		if sentinelExists() {
-			rebootRequiredGauge.WithLabelValues(nodeID).Set(1)
-		} else {
-			rebootRequiredGauge.WithLabelValues(nodeID).Set(0)
-		}
-		time.Sleep(time.Minute)
+	// Now call the rest of the main loop.
+	window, err := timewindow.New(rebootDays, rebootStart, rebootEnd, timezone)
+	if err != nil {
+		log.Fatalf("Failed to build time window: %v", err)
 	}
-}
+	log.Infof("Reboot schedule: %v", window)
 
-// nodeMeta is used to remember information across reboots
-type nodeMeta struct {
-	Unschedulable bool `json:"unschedulable"`
-}
+	log.Infof("Reboot method: %s", rebootMethod)
+	rebooter, err := internal.NewRebooter(rebootMethod, rebootCommand, rebootSignal)
+	if err != nil {
+		log.Fatalf("Failed to build rebooter: %v", err)
+	}
+
+	rebootChecker, err := internal.NewRebootChecker(rebootSentinelCommand, rebootSentinelFile)
+	if err != nil {
+		log.Fatalf("Failed to build reboot checker: %v", err)
+	}
 
-func rebootAsRequired(nodeID string) {
 	config, err := rest.InClusterConfig()
 	if err != nil {
 		log.Fatal(err)
@@ -232,55 +265,463 @@ func rebootAsRequired(nodeID string) {
 		log.Fatal(err)
 	}
 
-	lock := daemonsetlock.New(client, nodeID, dsNamespace, dsName, lockAnnotation)
-
-	nodeMeta := nodeMeta{}
-	if holding(lock, &nodeMeta) {
-		if !nodeMeta.Unschedulable {
-			uncordon(nodeID)
-		}
-		release(lock)
+	var blockCheckers []blockers.RebootBlocker
+	if prometheusURL != "" {
+		blockCheckers = append(blockCheckers, blockers.NewPrometheusBlockingChecker(papi.Config{Address: prometheusURL}, alertFilter.Regexp, alertFiringOnly, alertFilterMatchOnly))
 	}
-
-	source := rand.NewSource(time.Now().UnixNano())
-	tick := delaytick.New(source, period)
-	for _ = range tick {
-		if rebootRequired() && !rebootBlocked() {
-			node, err := client.CoreV1().Nodes().Get(nodeID, metav1.GetOptions{})
-			if err != nil {
-				log.Fatal(err)
-			}
-			nodeMeta.Unschedulable = node.Spec.Unschedulable
-
-			if acquire(lock, &nodeMeta) {
-				if !nodeMeta.Unschedulable {
-					drain(nodeID)
-				}
-				commandReboot(nodeID)
-				for {
-					log.Infof("Waiting for reboot")
-					time.Sleep(time.Minute)
-				}
-			}
-		}
+	if podSelectors != nil {
+		blockCheckers = append(blockCheckers, blockers.NewKubernetesBlockingChecker(client, nodeID, podSelectors))
 	}
-}
-
-func root(cmd *cobra.Command, args []string) {
-	log.Infof("Kubernetes Reboot Daemon: %s", version)
-
-	nodeID := os.Getenv("KURED_NODE_ID")
-	if nodeID == "" {
-		log.Fatal("KURED_NODE_ID environment variable required")
-	}
-
-	log.Infof("Node ID: %s", nodeID)
 	log.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation)
-	log.Infof("Reboot Sentinel: %s every %v", rebootSentinel, period)
+	if lockTTL > 0 {
+		log.Infof("Lock TTL set, lock will expire after: %v", lockTTL)
+	} else {
+		log.Info("Lock TTL not set, lock will remain until being released")
+	}
+	if lockReleaseDelay > 0 {
+		log.Infof("Lock release delay set, lock release will be delayed by: %v", lockReleaseDelay)
+	} else {
+		log.Info("Lock release delay not set, lock will be released immediately after rebooting")
+	}
+	lock := daemonsetlock.New(client, nodeID, dsNamespace, dsName, lockAnnotation, lockTTL, concurrency, lockReleaseDelay)
 
-	go rebootAsRequired(nodeID)
-	go maintainRebootRequiredMetric(nodeID)
+	go rebootAsRequired(nodeID, rebooter, rebootChecker, blockCheckers, window, lock, client)
+	go maintainRebootRequiredMetric(nodeID, rebootChecker)
 
 	http.Handle("/metrics", promhttp.Handler())
-	log.Fatal(http.ListenAndServe(":8080", nil))
+	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", metricsHost, metricsPort), nil)) // #nosec G114
+}
+
+func validateNodeLabels(preRebootNodeLabels []string, postRebootNodeLabels []string) error {
+	var preRebootNodeLabelKeys, postRebootNodeLabelKeys []string
+	for _, label := range preRebootNodeLabels {
+		preRebootNodeLabelKeys = append(preRebootNodeLabelKeys, strings.Split(label, "=")[0])
+	}
+	for _, label := range postRebootNodeLabels {
+		postRebootNodeLabelKeys = append(postRebootNodeLabelKeys, strings.Split(label, "=")[0])
+	}
+	sort.Strings(preRebootNodeLabelKeys)
+	sort.Strings(postRebootNodeLabelKeys)
+	if !reflect.DeepEqual(preRebootNodeLabelKeys, postRebootNodeLabelKeys) {
+		return fmt.Errorf("pre-reboot-node-labels keys and post-reboot-node-labels keys do not match, resulting in unexpected behaviour")
+	}
+
+	return nil
+}
+
+func validateNotificationURL(notifyURL string, slackHookURL string) string {
+	switch {
+	case slackHookURL != "" && notifyURL != "":
+		log.Warnf("Cannot use both --notify-url (given: %v) and --slack-hook-url (given: %v) flags. Kured will only use --notify-url flag", slackHookURL, notifyURL)
+		return validateNotificationURL(notifyURL, "")
+	case notifyURL != "":
+		return stripQuotes(notifyURL)
+	case slackHookURL != "":
+		log.Warnf("Deprecated flag(s). Please use --notify-url flag instead.")
+		parsedURL, err := url.Parse(stripQuotes(slackHookURL))
+		if err != nil {
+			log.Errorf("slack-hook-url is not properly formatted... no notification will be sent: %v\n", err)
+			return ""
+		}
+		if len(strings.Split(strings.ReplaceAll(parsedURL.Path, "/services/", ""), "/")) != 3 {
+			log.Errorf("slack-hook-url is not properly formatted... no notification will be sent: unexpected number of / in URL\n")
+			return ""
+		}
+		return fmt.Sprintf("slack://%s", strings.ReplaceAll(parsedURL.Path, "/services/", ""))
+	}
+	return ""
+}
+
+// LoadFromEnv attempts to load environment variables corresponding to flags.
+// It looks for an environment variable with the uppercase version of the flag name (prefixed by EnvPrefix).
+func LoadFromEnv() {
+	flag.VisitAll(func(f *flag.Flag) {
+		envVarName := fmt.Sprintf("%s_%s", EnvPrefix, strings.ToUpper(strings.ReplaceAll(f.Name, "-", "_")))
+
+		if envValue, exists := os.LookupEnv(envVarName); exists {
+			switch f.Value.Type() {
+			case "int":
+				if parsedVal, err := strconv.Atoi(envValue); err == nil {
+					err := flag.Set(f.Name, strconv.Itoa(parsedVal))
+					if err != nil {
+						fmt.Printf("cannot set flag %s from env var named %s", f.Name, envVarName)
+						os.Exit(1)
+					} // Set int flag
+				} else {
+					fmt.Printf("Invalid value for env var named %s", envVarName)
+					os.Exit(1)
+				}
+			case "string":
+				err := flag.Set(f.Name, envValue)
+				if err != nil {
+					fmt.Printf("cannot set flag %s from env{%s}: %s\n", f.Name, envVarName, envValue)
+					os.Exit(1)
+				} // Set string flag
+			case "bool":
+				if parsedVal, err := strconv.ParseBool(envValue); err == nil {
+					err := flag.Set(f.Name, strconv.FormatBool(parsedVal))
+					if err != nil {
+						fmt.Printf("cannot set flag %s from env{%s}: %s\n", f.Name, envVarName, envValue)
+						os.Exit(1)
+					} // Set boolean flag
+				} else {
+					fmt.Printf("Invalid value for %s: %s\n", envVarName, envValue)
+					os.Exit(1)
+				}
+			case "duration":
+				// Set duration from the environment variable (e.g., "1h30m")
+				if _, err := time.ParseDuration(envValue); err == nil {
+					err = flag.Set(f.Name, envValue)
+					if err != nil {
+						fmt.Printf("cannot set flag %s from env{%s}: %s\n", f.Name, envVarName, envValue)
+						os.Exit(1)
+					}
+				} else {
+					fmt.Printf("Invalid duration for %s: %s\n", envVarName, envValue)
+					os.Exit(1)
+				}
+			case "regexp":
+				// For regexp, set it from the environment variable
+				err := flag.Set(f.Name, envValue)
+				if err != nil {
+					fmt.Printf("cannot set flag %s from env{%s}: %s\n", f.Name, envVarName, envValue)
+					os.Exit(1)
+				}
+			case "stringSlice":
+				// For stringSlice, split the environment variable by commas and set it
+				err := flag.Set(f.Name, envValue)
+				if err != nil {
+					fmt.Printf("cannot set flag %s from env{%s}: %s\n", f.Name, envVarName, envValue)
+					os.Exit(1)
+				}
+			default:
+				fmt.Printf("Unsupported flag type for %s\n", f.Name)
+			}
+		}
+	})
+
+}
+
+// stripQuotes removes any literal single or double quote chars that surround a string
+func stripQuotes(str string) string {
+	if len(str) > 2 {
+		firstChar := str[0]
+		lastChar := str[len(str)-1]
+		if firstChar == lastChar && (firstChar == '"' || firstChar == '\'') {
+			return str[1 : len(str)-1]
+		}
+	}
+	// return the original string if it has a length of zero or one
+	return str
+}
+
+func drain(client *kubernetes.Clientset, node *v1.Node) error {
+	nodename := node.GetName()
+
+	if preRebootNodeLabels != nil {
+		updateNodeLabels(client, node, preRebootNodeLabels)
+	}
+
+	if drainDelay > 0 {
+		log.Infof("Delaying drain for %v", drainDelay)
+		time.Sleep(drainDelay)
+	}
+
+	log.Infof("Draining node %s", nodename)
+
+	if notifyURL != "" {
+		if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateDrain, nodename)); err != nil {
+			log.Warnf("Error notifying: %v", err)
+		}
+	}
+
+	drainer := &kubectldrain.Helper{
+		Client:                          client,
+		Ctx:                             context.Background(),
+		GracePeriodSeconds:              drainGracePeriod,
+		PodSelector:                     drainPodSelector,
+		SkipWaitForDeleteTimeoutSeconds: skipWaitForDeleteTimeoutSeconds,
+		Force:                           true,
+		DeleteEmptyDirData:              true,
+		IgnoreAllDaemonSets:             true,
+		ErrOut:                          os.Stderr,
+		Out:                             os.Stdout,
+		Timeout:                         drainTimeout,
+	}
+
+	if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil {
+		log.Errorf("Error cordonning %s: %v", nodename, err)
+		return err
+	}
+
+	if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil {
+		log.Errorf("Error draining %s: %v", nodename, err)
+		return err
+	}
+	return nil
+}
+
+func uncordon(client *kubernetes.Clientset, node *v1.Node) error {
+	nodename := node.GetName()
+	log.Infof("Uncordoning node %s", nodename)
+	drainer := &kubectldrain.Helper{
+		Client: client,
+		ErrOut: os.Stderr,
+		Out:    os.Stdout,
+		Ctx:    context.Background(),
+	}
+	if err := kubectldrain.RunCordonOrUncordon(drainer, node, false); err != nil {
+		log.Fatalf("Error uncordonning %s: %v", nodename, err)
+		return err
+	} else if postRebootNodeLabels != nil {
+		updateNodeLabels(client, node, postRebootNodeLabels)
+	}
+	return nil
+}
+
+func maintainRebootRequiredMetric(nodeID string, checker checkers.Checker) {
+	for {
+		if checker.RebootRequired() {
+			rebootRequiredGauge.WithLabelValues(nodeID).Set(1)
+		} else {
+			rebootRequiredGauge.WithLabelValues(nodeID).Set(0)
+		}
+		time.Sleep(time.Minute)
+	}
+}
+
+func addNodeAnnotations(client *kubernetes.Clientset, nodeID string, annotations map[string]string) error {
+	node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+	if err != nil {
+		log.Errorf("Error retrieving node object via k8s API: %s", err)
+		return err
+	}
+	for k, v := range annotations {
+		node.Annotations[k] = v
+		log.Infof("Adding node %s annotation: %s=%s", node.GetName(), k, v)
+	}
+
+	bytes, err := json.Marshal(node)
+	if err != nil {
+		log.Errorf("Error marshalling node object into JSON: %v", err)
+		return err
+	}
+
+	_, err = client.CoreV1().Nodes().Patch(context.TODO(), node.GetName(), types.StrategicMergePatchType, bytes, metav1.PatchOptions{})
+	if err != nil {
+		var annotationsErr string
+		for k, v := range annotations {
+			annotationsErr += fmt.Sprintf("%s=%s ", k, v)
+		}
+		log.Errorf("Error adding node annotations %s via k8s API: %v", annotationsErr, err)
+		return err
+	}
+	return nil
+}
+
+func deleteNodeAnnotation(client *kubernetes.Clientset, nodeID, key string) error {
+	log.Infof("Deleting node %s annotation %s", nodeID, key)
+
+	// JSON Patch takes as path input a JSON Pointer, defined in RFC6901
+	// So we replace all instances of "/" with "~1" as per:
+	// https://tools.ietf.org/html/rfc6901#section-3
+	patch := []byte(fmt.Sprintf("[{\"op\":\"remove\",\"path\":\"/metadata/annotations/%s\"}]", strings.ReplaceAll(key, "/", "~1")))
+	_, err := client.CoreV1().Nodes().Patch(context.TODO(), nodeID, types.JSONPatchType, patch, metav1.PatchOptions{})
+	if err != nil {
+		log.Errorf("Error deleting node annotation %s via k8s API: %v", key, err)
+		return err
+	}
+	return nil
+}
+
+func updateNodeLabels(client *kubernetes.Clientset, node *v1.Node, labels []string) {
+	labelsMap := make(map[string]string)
+	for _, label := range labels {
+		k := strings.Split(label, "=")[0]
+		v := strings.Split(label, "=")[1]
+		labelsMap[k] = v
+		log.Infof("Updating node %s label: %s=%s", node.GetName(), k, v)
+	}
+
+	bytes, err := json.Marshal(map[string]interface{}{
+		"metadata": map[string]interface{}{
+			"labels": labelsMap,
+		},
+	})
+	if err != nil {
+		log.Fatalf("Error marshalling node object into JSON: %v", err)
+	}
+
+	_, err = client.CoreV1().Nodes().Patch(context.TODO(), node.GetName(), types.StrategicMergePatchType, bytes, metav1.PatchOptions{})
+	if err != nil {
+		var labelsErr string
+		for _, label := range labels {
+			k := strings.Split(label, "=")[0]
+			v := strings.Split(label, "=")[1]
+			labelsErr += fmt.Sprintf("%s=%s ", k, v)
+		}
+		log.Errorf("Error updating node labels %s via k8s API: %v", labelsErr, err)
+	}
+}
+
+func rebootAsRequired(nodeID string, rebooter reboot.Rebooter, checker checkers.Checker, blockCheckers []blockers.RebootBlocker, window *timewindow.TimeWindow, lock daemonsetlock.Lock, client *kubernetes.Clientset) {
+
+	source := rand.NewSource(time.Now().UnixNano())
+	tick := delaytick.New(source, 1*time.Minute)
+	for range tick {
+		holding, lockData, err := lock.Holding()
+		if err != nil {
+			log.Errorf("Error testing lock: %v", err)
+		}
+		if holding {
+			node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+			if err != nil {
+				log.Errorf("Error retrieving node object via k8s API: %v", err)
+				continue
+			}
+
+			if !lockData.Metadata.Unschedulable {
+				err = uncordon(client, node)
+				if err != nil {
+					log.Errorf("Unable to uncordon %s: %v, will continue to hold lock and retry uncordon", node.GetName(), err)
+					continue
+				}
+
+				if notifyURL != "" {
+					if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateUncordon, nodeID)); err != nil {
+						log.Warnf("Error notifying: %v", err)
+					}
+				}
+			}
+			// If we're holding the lock we know we've tried, in a prior run, to reboot
+			// So (1) we want to confirm that the reboot succeeded practically ( !rebootRequired() )
+			// And (2) check if we previously annotated the node that it was in the process of being rebooted,
+			// And finally (3) if it has that annotation, to delete it.
+			// This indicates to other node tools running on the cluster that this node may be a candidate for maintenance
+			if annotateNodes && !checker.RebootRequired() {
+				if _, ok := node.Annotations[KuredRebootInProgressAnnotation]; ok {
+					err := deleteNodeAnnotation(client, nodeID, KuredRebootInProgressAnnotation)
+					if err != nil {
+						continue
+					}
+				}
+			}
+
+			err = lock.Release()
+			if err != nil {
+				log.Errorf("Error releasing lock, will retry: %v", err)
+				continue
+			}
+		}
+		break
+	}
+
+	preferNoScheduleTaint := taints.New(client, nodeID, preferNoScheduleTaintName, v1.TaintEffectPreferNoSchedule)
+
+	// Remove taint immediately during startup to quickly allow scheduling again.
+	if !checker.RebootRequired() {
+		preferNoScheduleTaint.Disable()
+	}
+
+	source = rand.NewSource(time.Now().UnixNano())
+	tick = delaytick.New(source, period)
+	for range tick {
+		if !window.Contains(time.Now()) {
+			// Remove taint outside the reboot time window to allow for normal operation.
+			preferNoScheduleTaint.Disable()
+			continue
+		}
+
+		if !checker.RebootRequired() {
+			log.Infof("Reboot not required")
+			preferNoScheduleTaint.Disable()
+			continue
+		}
+
+		node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+		if err != nil {
+			log.Fatalf("Error retrieving node object via k8s API: %v", err)
+		}
+
+		nodeMeta := daemonsetlock.NodeMeta{Unschedulable: node.Spec.Unschedulable}
+
+		var timeNowString string
+		if annotateNodes {
+			if _, ok := node.Annotations[KuredRebootInProgressAnnotation]; !ok {
+				timeNowString = time.Now().Format(time.RFC3339)
+				// Annotate this node to indicate that "I am going to be rebooted!"
+				// so that other node maintenance tools running on the cluster are aware that this node is in the process of a "state transition"
+				annotations := map[string]string{KuredRebootInProgressAnnotation: timeNowString}
+				// & annotate this node with a timestamp so that other node maintenance tools know how long it's been since this node has been marked for reboot
+				annotations[KuredMostRecentRebootNeededAnnotation] = timeNowString
+				err := addNodeAnnotations(client, nodeID, annotations)
+				if err != nil {
+					continue
+				}
+			}
+		}
+
+		var rebootRequiredBlockCondition string
+		if blockers.RebootBlocked(blockCheckers...) {
+			rebootRequiredBlockCondition = ", but blocked at this time"
+			continue
+		}
+		log.Infof("Reboot required%s", rebootRequiredBlockCondition)
+
+		holding, _, err := lock.Holding()
+		if err != nil {
+			log.Errorf("Error testing lock: %v", err)
+		}
+
+		if !holding {
+			acquired, holder, err := lock.Acquire(nodeMeta)
+			if err != nil {
+				log.Errorf("Error acquiring lock: %v", err)
+			}
+			if !acquired {
+				log.Warnf("Lock already held: %v", holder)
+				// Prefer to not schedule pods onto this node to avoid draing the same pod multiple times.
+				preferNoScheduleTaint.Enable()
+				continue
+			}
+		}
+
+		err = drain(client, node)
+		if err != nil {
+			if !forceReboot {
+				log.Errorf("Unable to cordon or drain %s: %v, will release lock and retry cordon and drain before rebooting when lock is next acquired", node.GetName(), err)
+				err = lock.Release()
+				if err != nil {
+					log.Errorf("Error releasing lock: %v", err)
+				}
+				log.Infof("Performing a best-effort uncordon after failed cordon and drain")
+				err := uncordon(client, node)
+				if err != nil {
+					log.Errorf("Failed to uncordon %s: %v", node.GetName(), err)
+				}
+				continue
+			}
+		}
+
+		if rebootDelay > 0 {
+			log.Infof("Delaying reboot for %v", rebootDelay)
+			time.Sleep(rebootDelay)
+		}
+
+		if notifyURL != "" {
+			if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateReboot, nodeID)); err != nil {
+				log.Warnf("Error notifying: %v", err)
+			}
+		}
+		log.Infof("Triggering reboot for node %v", nodeID)
+
+		err = rebooter.Reboot()
+		if err != nil {
+			log.Fatalf("Unable to reboot node: %v", err)
+		}
+		for {
+			log.Infof("Waiting for reboot")
+			time.Sleep(time.Minute)
+		}
+	}
 }

cmd/kured/main_test.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestValidateNotificationURL(t *testing.T) {
+
+	tests := []struct {
+		name         string
+		slackHookURL string
+		notifyURL    string
+		expected     string
+	}{
+		{"slackHookURL only works fine", "https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET", "", "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"},
+		{"slackHookURL and notify URL together only keeps notifyURL", "\"https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET\"", "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com", "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com"},
+		{"slackHookURL removes extraneous double quotes", "\"https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET\"", "", "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"},
+		{"slackHookURL removes extraneous single quotes", "'https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET'", "", "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"},
+		{"notifyURL removes extraneous double quotes", "", "\"teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com\"", "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com"},
+		{"notifyURL removes extraneous single quotes", "", "'teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com'", "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := validateNotificationURL(tt.notifyURL, tt.slackHookURL); !reflect.DeepEqual(got, tt.expected) {
+				t.Errorf("validateNotificationURL() = %v, expected %v", got, tt.expected)
+			}
+		})
+	}
+}
+
+func Test_stripQuotes(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "string with no surrounding quotes is unchanged",
+			input:    "Hello, world!",
+			expected: "Hello, world!",
+		},
+		{
+			name:     "string with surrounding double quotes should strip quotes",
+			input:    "\"Hello, world!\"",
+			expected: "Hello, world!",
+		},
+		{
+			name:     "string with surrounding single quotes should strip quotes",
+			input:    "'Hello, world!'",
+			expected: "Hello, world!",
+		},
+		{
+			name:     "string with unbalanced surrounding quotes is unchanged",
+			input:    "'Hello, world!\"",
+			expected: "'Hello, world!\"",
+		},
+		{
+			name:     "string with length of one is unchanged",
+			input:    "'",
+			expected: "'",
+		},
+		{
+			name:     "string with length of zero is unchanged",
+			input:    "",
+			expected: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := stripQuotes(tt.input); !reflect.DeepEqual(got, tt.expected) {
+				t.Errorf("stripQuotes() = %v, expected %v", got, tt.expected)
+			}
+		})
+	}
+}

cmd/kured/pflags.go

@@ -5,14 +5,14 @@ import (
 )
 
 type regexpValue struct {
-	value **regexp.Regexp
+	*regexp.Regexp
 }
 
 func (rev *regexpValue) String() string {
-	if *rev.value == nil {
+	if rev.Regexp == nil {
 		return ""
 	}
-	return (*rev.value).String()
+	return rev.Regexp.String()
 }
 
 func (rev *regexpValue) Set(s string) error {
@@ -20,12 +20,11 @@ func (rev *regexpValue) Set(s string) error {
 	if err != nil {
 		return err
 	}
-
-	*rev.value = value
-
+	rev.Regexp = value
 	return nil
 }
 
+// Type method returns the type of the flag as a string
 func (rev *regexpValue) Type() string {
-	return "regexp.Regexp"
+	return "regexp"
 }

cmd/prom-active-alerts/main.go

@@ -1,23 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"regexp"
-
-	"github.com/weaveworks/kured/pkg/alerts"
-)
-
-func main() {
-	if len(os.Args) != 3 {
-		log.Fatalf("USAGE: %s <prometheusURL> <filterRegexp>", os.Args[0])
-	}
-
-	count, err := alerts.PrometheusCountActive(os.Args[1], regexp.MustCompile(os.Args[2]))
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	fmt.Println(count)
-}

go.mod

@@ -0,0 +1,90 @@
+module github.com/kubereboot/kured
+
+go 1.24.11
+
+require (
+	github.com/containrrr/shoutrrr v0.8.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/common v0.67.5
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/pflag v1.0.10
+	github.com/stretchr/testify v1.11.1
+	k8s.io/api v0.34.3
+	k8s.io/apimachinery v0.34.3
+	k8s.io/client-go v0.34.3
+	k8s.io/kubectl v0.34.3
+)
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
+	github.com/fatih/color v1.15.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/btree v1.1.3 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/moby/spdystream v0.5.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/spf13/cobra v1.9.1 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xlab/treeprint v1.2.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.3 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/net v0.48.0 // indirect
+	golang.org/x/oauth2 v0.34.0 // indirect
+	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sys v0.39.0 // indirect
+	golang.org/x/term v0.38.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/cli-runtime v0.34.3 // indirect
+	k8s.io/component-base v0.34.3 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
+	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/kustomize/api v0.20.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
+)

go.sum

@@ -0,0 +1,263 @@
+github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
+github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
+github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
+github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/containrrr/shoutrrr v0.8.0 h1:mfG2ATzIS7NR2Ec6XL+xyoHzN97H8WPjir8aYzJUSec=
+github.com/containrrr/shoutrrr v0.8.0/go.mod h1:ioyQAyu1LJY6sILuNyKaQaw+9Ttik5QePU8atnAdO2o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU=
+github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4=
+github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
+github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
+github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
+github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
+github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
+github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
+github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
+github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
+github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jarcoal/httpmock v1.3.0 h1:2RJ8GP0IIaWwcC9Fp2BmVi8Kog3v2Hn7VXM3fTd+nuc=
+github.com/jarcoal/httpmock v1.3.0/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
+github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
+github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
+github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU=
+github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
+github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
+github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM=
+github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
+github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
+github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
+github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
+github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
+github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
+github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
+github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
+go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
+golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
+golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
+golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
+golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
+golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
+gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+k8s.io/api v0.34.3 h1:D12sTP257/jSH2vHV2EDYrb16bS7ULlHpdNdNhEw2S4=
+k8s.io/api v0.34.3/go.mod h1:PyVQBF886Q5RSQZOim7DybQjAbVs8g7gwJNhGtY5MBk=
+k8s.io/apimachinery v0.34.3 h1:/TB+SFEiQvN9HPldtlWOTp0hWbJ+fjU+wkxysf/aQnE=
+k8s.io/apimachinery v0.34.3/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw=
+k8s.io/cli-runtime v0.34.3 h1:YRyMhiwX0dT9lmG0AtZDaeG33Nkxgt9OlCTZhRXj9SI=
+k8s.io/cli-runtime v0.34.3/go.mod h1:GVwL1L5uaGEgM7eGeKjaTG2j3u134JgG4dAI6jQKhMc=
+k8s.io/client-go v0.34.3 h1:wtYtpzy/OPNYf7WyNBTj3iUA0XaBHVqhv4Iv3tbrF5A=
+k8s.io/client-go v0.34.3/go.mod h1:OxxeYagaP9Kdf78UrKLa3YZixMCfP6bgPwPwNBQBzpM=
+k8s.io/component-base v0.34.3 h1:zsEgw6ELqK0XncCQomgO9DpUIzlrYuZYA0Cgo+JWpVk=
+k8s.io/component-base v0.34.3/go.mod h1:5iIlD8wPfWE/xSHTRfbjuvUul2WZbI2nOUK65XL0E/c=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA=
+k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts=
+k8s.io/kubectl v0.34.3 h1:vpM6//153gh5gvsYHXWHVJ4l4xmN5QFwTSmlfd8icm8=
+k8s.io/kubectl v0.34.3/go.mod h1:zZQHtIZoUqTP1bAnPzq/3W1jfc0NeOeunFgcswrfg1c=
+k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y=
+k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
+sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/kustomize/api v0.20.1 h1:iWP1Ydh3/lmldBnH/S5RXgT98vWYMaTUL1ADcr+Sv7I=
+sigs.k8s.io/kustomize/api v0.20.1/go.mod h1:t6hUFxO+Ph0VxIk1sKp1WS0dOjbPCtLJ4p8aADLwqjM=
+sigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A78=
+sigs.k8s.io/kustomize/kyaml v0.20.1/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

internal/validators.go

@@ -0,0 +1,41 @@
+// Package internal provides convenient tools which shouldn't be in cmd/main
+// It will eventually provide internal validation and chaining logic to select
+// appropriate reboot and sentinel check methods based on configuration.
+// It validates user input and instantiates the correct checker and rebooter implementations
+// for use elsewhere in kured.
+package internal
+
+import (
+	"fmt"
+
+	"github.com/kubereboot/kured/pkg/checkers"
+	"github.com/kubereboot/kured/pkg/reboot"
+	log "github.com/sirupsen/logrus"
+)
+
+// NewRebooter validates the rebootMethod, rebootCommand, and rebootSignal input,
+// then chains to the right constructor.
+func NewRebooter(rebootMethod string, rebootCommand string, rebootSignal int) (reboot.Rebooter, error) {
+	switch rebootMethod {
+	case "command":
+		log.Infof("Reboot command: %s", rebootCommand)
+		return reboot.NewCommandRebooter(rebootCommand)
+	case "signal":
+		log.Infof("Reboot signal: %d", rebootSignal)
+		return reboot.NewSignalRebooter(rebootSignal)
+	default:
+		return nil, fmt.Errorf("invalid reboot-method configured %s, expected signal or command", rebootMethod)
+	}
+}
+
+// NewRebootChecker validates the rebootSentinelCommand, rebootSentinelFile input,
+// then chains to the right constructor.
+func NewRebootChecker(rebootSentinelCommand string, rebootSentinelFile string) (checkers.Checker, error) {
+	// An override of rebootSentinelCommand means a privileged command
+	if rebootSentinelCommand != "" {
+		log.Infof("Sentinel checker is (privileged) user provided command: %s", rebootSentinelCommand)
+		return checkers.NewCommandChecker(rebootSentinelCommand, 1, true)
+	}
+	log.Infof("Sentinel checker is (unprivileged) testing for the presence of: %s", rebootSentinelFile)
+	return checkers.NewFileRebootChecker(rebootSentinelFile)
+}

kured-ds-signal.yaml

@@ -0,0 +1,100 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kured
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kured # Must match `--ds-name`
+  namespace: kube-system # Must match `--ds-namespace`
+spec:
+  selector:
+    matchLabels:
+      name: kured
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        name: kured
+    spec:
+      serviceAccountName: kured
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      hostPID: true # Facilitate entering the host mount namespace via init
+      restartPolicy: Always
+      volumes:
+        - name: sentinel
+          hostPath:
+            path: /var/run
+            type: Directory
+      containers:
+        - name: kured
+          # If you find yourself here wondering why there is no
+          # :latest tag on Docker Hub,see the FAQ in the README
+          image: ghcr.io/kubereboot/kured:1.21.0
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            privileged: false # Give permission to nsenter /proc/1/ns/mnt
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["*"]
+              add: ["CAP_KILL"]
+          ports:
+            - containerPort: 8080
+              name: metrics
+          env:
+            # Pass in the name of the node on which this pod is scheduled
+            # for use with drain/uncordon operations and lock acquisition
+            - name: KURED_NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /sentinel
+              name: sentinel
+              readOnly: true
+          command:
+            - /usr/bin/kured
+            - --reboot-sentinel=/sentinel/reboot-required
+            - --reboot-method=signal
+#            - --reboot-signal=39
+#            - --force-reboot=false
+#            - --drain-grace-period=-1
+#            - --skip-wait-for-delete-timeout=0
+#            - --drain-timeout=0
+#            - --period=1h
+#            - --ds-namespace=kube-system
+#            - --ds-name=kured
+#            - --lock-annotation=weave.works/kured-node-lock
+#            - --lock-ttl=0
+#            - --prometheus-url=http://prometheus.monitoring.svc.cluster.local
+#            - --alert-filter-regexp=^RebootRequired$
+#            - --alert-firing-only=false
+#            - --prefer-no-schedule-taint=""
+#            - --reboot-sentinel-command=""
+#            - --slack-hook-url=https://hooks.slack.com/...
+#            - --slack-username=prod
+#            - --slack-channel=alerting
+#            - --notify-url="" # See also shoutrrr url format
+#            - --message-template-drain=Draining node %s
+#            - --message-template-reboot=Rebooting node %s
+#            - --message-template-uncordon=Node %s rebooted & uncordoned successfully!
+#            - --blocking-pod-selector=runtime=long,cost=expensive
+#            - --blocking-pod-selector=name=temperamental
+#            - --blocking-pod-selector=...
+#            - --reboot-days=sun,mon,tue,wed,thu,fri,sat
+#            - --reboot-delay=90s
+#            - --start-time=0:00
+#            - --end-time=23:59:59
+#            - --time-zone=UTC
+#            - --annotate-nodes=false
+#            - --lock-release-delay=30m
+#            - --log-format=text

kured-ds.yaml

@@ -1,32 +1,51 @@
-apiVersion: extensions/v1beta1
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kured
+  namespace: kube-system
+---
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: kured            # Must match `--ds-name`
+  name: kured # Must match `--ds-name`
   namespace: kube-system # Must match `--ds-namespace`
 spec:
+  selector:
+    matchLabels:
+      name: kured
+  updateStrategy:
+    type: RollingUpdate
   template:
     metadata:
       labels:
         name: kured
     spec:
+      serviceAccountName: kured
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      hostPID: true # Facilitate entering the host mount namespace via init
+      restartPolicy: Always
+      volumes:
+        - name: sentinel
+          hostPath:
+            path: /var/run
+            type: Directory
       containers:
         - name: kured
-          image: quay.io/weaveworks/kured
+          # If you find yourself here wondering why there is no
+          # :latest tag on Docker Hub,see the FAQ in the README
+          image: ghcr.io/kubereboot/kured:1.21.0
           imagePullPolicy: IfNotPresent
-          command:
-            - /usr/bin/kured
-#          args:
-#            - --alert-filter-regexp=^RebootRequired$
-#            - --ds-name=kured
-#            - --ds-namespace=kube-system
-#            - --lock-annotation=weave.works/kured-node-lock
-#            - --period=1h
-#            - --prometheus-url=http://prometheus.monitoring.svc.cluster.local
-#            - --reboot-sentinel=/var/run/reboot-required
-#            - --slack-hook-url=https://hooks.slack.com/...
-#            - --slack-username=prod
-#
-# NO USER SERVICEABLE PARTS BEYOND THIS POINT
+          securityContext:
+            privileged: true # Give permission to nsenter /proc/1/ns/mnt
+            readOnlyRootFilesystem: true
+          ports:
+            - containerPort: 8080
+              name: metrics
           env:
             # Pass in the name of the node on which this pod is scheduled
             # for use with drain/uncordon operations and lock acquisition
@@ -35,13 +54,49 @@ spec:
                 fieldRef:
                   fieldPath: spec.nodeName
           volumeMounts:
-            # Needed for two purposes:
-            # * Testing for the existence of /var/run/reboot-required
-            # * Accessing /var/run/dbus/system_bus_socket to effect reboot
-            - name: hostrun
-              mountPath: /var/run
-      restartPolicy: Always
-      volumes:
-        - name: hostrun
-          hostPath:
-            path: /var/run
+            - mountPath: /sentinel
+              name: sentinel
+              readOnly: true
+          command:
+            - /usr/bin/kured
+            - --reboot-sentinel=/sentinel/reboot-required
+#            - --force-reboot=false
+#            - --drain-grace-period=-1
+#            - --skip-wait-for-delete-timeout=0
+#            - --drain-delay=0
+#            - --drain-timeout=0
+#            - --drain-pod-selector=""
+#            - --period=1h
+#            - --ds-namespace=kube-system
+#            - --ds-name=kured
+#            - --lock-annotation=weave.works/kured-node-lock
+#            - --lock-ttl=0
+#            - --prometheus-url=http://prometheus.monitoring.svc.cluster.local
+#            - --alert-filter-regexp=^RebootRequired$
+#            - --alert-filter-match-only=false
+#            - --alert-firing-only=false
+#            - --prefer-no-schedule-taint=""
+#            - --reboot-sentinel-command=""
+#            - --reboot-method=command
+#            - --reboot-signal=39
+#            - --slack-hook-url=https://hooks.slack.com/...
+#            - --slack-username=prod
+#            - --slack-channel=alerting
+#            - --notify-url="" # See also shoutrrr url format
+#            - --message-template-drain=Draining node %s
+#            - --message-template-reboot=Rebooting node %s
+#            - --message-template-uncordon=Node %s rebooted & uncordoned successfully!
+#            - --blocking-pod-selector=runtime=long,cost=expensive
+#            - --blocking-pod-selector=name=temperamental
+#            - --blocking-pod-selector=...
+#            - --reboot-days=sun,mon,tue,wed,thu,fri,sat
+#            - --reboot-delay=90s
+#            - --start-time=0:00
+#            - --end-time=23:59:59
+#            - --time-zone=UTC
+#            - --annotate-nodes=false
+#            - --lock-release-delay=30m
+#            - --log-format=text
+#            - --metrics-host=""
+#            - --metrics-port=8080
+#            - --concurrency=1

kured-rbac.yaml

@@ -0,0 +1,63 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kured
+rules:
+# Allow kured to read spec.unschedulable
+# Allow kubectl to drain/uncordon
+#
+# NB: These permissions are tightly coupled to the bundled version of kubectl; the ones below
+# match https://github.com/kubernetes/kubernetes/blob/v1.19.4/staging/src/k8s.io/kubectl/pkg/cmd/drain/drain.go
+#
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs:     ["get", "patch"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs:     ["list","delete","get"]
+- apiGroups: ["apps"]
+  resources: ["daemonsets"]
+  verbs:     ["get"]
+- apiGroups: [""]
+  resources: ["pods/eviction"]
+  verbs:     ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kured
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kured
+subjects:
+- kind: ServiceAccount
+  name: kured
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: kube-system
+  name: kured
+rules:
+# Allow kured to lock/unlock itself
+- apiGroups:     ["apps"]
+  resources:     ["daemonsets"]
+  resourceNames: ["kured"]
+  verbs:         ["update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: kube-system
+  name: kured
+subjects:
+- kind: ServiceAccount
+  namespace: kube-system
+  name: kured
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kured

pkg/alerts/prometheus.go

@@ -1,51 +0,0 @@
-package alerts
-
-import (
-	"context"
-	"fmt"
-	"regexp"
-	"sort"
-	"time"
-
-	"github.com/prometheus/client_golang/api/prometheus"
-	"github.com/prometheus/common/model"
-)
-
-// Returns a list of names of active (e.g. pending or firing) alerts, filtered
-// by the supplied regexp.
-func PrometheusActiveAlerts(prometheusURL string, filter *regexp.Regexp) ([]string, error) {
-	client, err := prometheus.New(prometheus.Config{Address: prometheusURL})
-	if err != nil {
-		return nil, err
-	}
-
-	queryAPI := prometheus.NewQueryAPI(client)
-
-	value, err := queryAPI.Query(context.Background(), "ALERTS", time.Now())
-	if err != nil {
-		return nil, err
-	}
-
-	if value.Type() == model.ValVector {
-		if vector, ok := value.(model.Vector); ok {
-			activeAlertSet := make(map[string]bool)
-			for _, sample := range vector {
-				if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 {
-					if filter == nil || !filter.MatchString(string(alertName)) {
-						activeAlertSet[string(alertName)] = true
-					}
-				}
-			}
-
-			var activeAlerts []string
-			for activeAlert, _ := range activeAlertSet {
-				activeAlerts = append(activeAlerts, activeAlert)
-			}
-			sort.Sort(sort.StringSlice(activeAlerts))
-
-			return activeAlerts, nil
-		}
-	}
-
-	return nil, fmt.Errorf("Unexpected value type: %v", value)
-}

pkg/blockers/blockers.go

@@ -0,0 +1,21 @@
+// Package blockers provides interfaces and implementations for determining
+// whether a system should be prevented to reboot.
+// You can use that package if you fork Kured's main loop.
+package blockers
+
+// RebootBlocked checks that a single block Checker
+// will block the reboot or not.
+func RebootBlocked(blockers ...RebootBlocker) bool {
+	for _, blocker := range blockers {
+		if blocker.IsBlocked() {
+			return true
+		}
+	}
+	return false
+}
+
+// RebootBlocker interface should be implemented by types
+// to know if their instantiations should block a reboot
+type RebootBlocker interface {
+	IsBlocked() bool
+}

pkg/blockers/blockers_test.go

@@ -0,0 +1,65 @@
+package blockers
+
+import (
+	papi "github.com/prometheus/client_golang/api"
+	"testing"
+)
+
+type BlockingChecker struct {
+	blocking bool
+}
+
+func (fbc BlockingChecker) IsBlocked() bool {
+	return fbc.blocking
+}
+
+func Test_rebootBlocked(t *testing.T) {
+	noCheckers := []RebootBlocker{}
+	nonblockingChecker := BlockingChecker{blocking: false}
+	blockingChecker := BlockingChecker{blocking: true}
+
+	// Instantiate a prometheusClient with a broken_url
+	brokenPrometheusClient := NewPrometheusBlockingChecker(papi.Config{Address: "broken_url"}, nil, false, false)
+
+	type args struct {
+		blockers []RebootBlocker
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Do not block on no blocker defined",
+			args: args{blockers: noCheckers},
+			want: false,
+		},
+		{
+			name: "Ensure a blocker blocks",
+			args: args{blockers: []RebootBlocker{blockingChecker}},
+			want: true,
+		},
+		{
+			name: "Ensure a non-blocker doesn't block",
+			args: args{blockers: []RebootBlocker{nonblockingChecker}},
+			want: false,
+		},
+		{
+			name: "Ensure one blocker is enough to block",
+			args: args{blockers: []RebootBlocker{nonblockingChecker, blockingChecker}},
+			want: true,
+		},
+		{
+			name: "Do block on error contacting prometheus API",
+			args: args{blockers: []RebootBlocker{brokenPrometheusClient}},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := RebootBlocked(tt.args.blockers...); got != tt.want {
+				t.Errorf("rebootBlocked() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/blockers/kubernetespod.go

@@ -0,0 +1,64 @@
+package blockers
+
+import (
+	"context"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Compile-time checks to ensure the type implements the interface
+var (
+	_ RebootBlocker = (*KubernetesBlockingChecker)(nil)
+)
+
+// KubernetesBlockingChecker contains info for connecting
+// to k8s, and can give info about whether a reboot should be blocked
+type KubernetesBlockingChecker struct {
+	// client used to contact kubernetes API
+	client   *kubernetes.Clientset
+	nodeName string
+	// lised used to filter pods (podSelector)
+	filter []string
+}
+
+// NewKubernetesBlockingChecker creates a new KubernetesBlockingChecker using the provided Kubernetes client,
+// node name, and pod selectors.
+func NewKubernetesBlockingChecker(client *kubernetes.Clientset, nodename string, podSelectors []string) *KubernetesBlockingChecker {
+	return &KubernetesBlockingChecker{
+		client:   client,
+		nodeName: nodename,
+		filter:   podSelectors,
+	}
+}
+
+// IsBlocked for the KubernetesBlockingChecker will check if a pod, for the node, is preventing
+// the reboot. It will warn in the logs about blocking, but does not return an error.
+func (kb KubernetesBlockingChecker) IsBlocked() bool {
+	fieldSelector := fmt.Sprintf("spec.nodeName=%s,status.phase!=Succeeded,status.phase!=Failed,status.phase!=Unknown", kb.nodeName)
+	for _, labelSelector := range kb.filter {
+		podList, err := kb.client.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
+			LabelSelector: labelSelector,
+			FieldSelector: fieldSelector,
+			Limit:         10})
+		if err != nil {
+			log.Warnf("Reboot blocked: pod query error: %v", err)
+			return true
+		}
+
+		if len(podList.Items) > 0 {
+			podNames := make([]string, 0, len(podList.Items))
+			for _, pod := range podList.Items {
+				podNames = append(podNames, pod.Name)
+			}
+			if len(podList.Continue) > 0 {
+				podNames = append(podNames, "...")
+			}
+			log.Warnf("Reboot blocked: matching pods: %v", podNames)
+			return true
+		}
+	}
+	return false
+}

pkg/blockers/prometheus.go

@@ -0,0 +1,121 @@
+package blockers
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"sort"
+	"time"
+
+	papi "github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/common/model"
+	log "github.com/sirupsen/logrus"
+)
+
+// Compile-time checks to ensure the type implements the interface
+var (
+	_ RebootBlocker = (*PrometheusBlockingChecker)(nil)
+)
+
+// PrometheusBlockingChecker contains info for connecting
+// to prometheus, and can give info about whether a reboot should be blocked
+type PrometheusBlockingChecker struct {
+	promConfig papi.Config
+	// regexp used to get alerts
+	filter *regexp.Regexp
+	// bool to indicate if only firing alerts should be considered
+	firingOnly bool
+	// bool to indicate that we're only blocking on alerts which match the filter
+	filterMatchOnly bool
+	// storing the promClient
+	promClient papi.Client
+}
+
+// NewPrometheusBlockingChecker creates a new PrometheusBlockingChecker using the given
+// Prometheus API config, alert filter, and filtering options.
+func NewPrometheusBlockingChecker(config papi.Config, alertFilter *regexp.Regexp, firingOnly bool, filterMatchOnly bool) PrometheusBlockingChecker {
+	promClient, _ := papi.NewClient(config)
+
+	return PrometheusBlockingChecker{
+		promConfig:      config,
+		filter:          alertFilter,
+		firingOnly:      firingOnly,
+		filterMatchOnly: filterMatchOnly,
+		promClient:      promClient,
+	}
+}
+
+// IsBlocked for the prometheus will check if there are active alerts matching
+// the arguments given into the PrometheusBlockingChecker which would actively
+// block the reboot.
+// As of today, no blocker information is shared as a return of the method,
+// and the information is simply logged.
+func (pb PrometheusBlockingChecker) IsBlocked() bool {
+	alertNames, err := pb.ActiveAlerts()
+	if err != nil {
+		log.Warnf("Reboot blocked: prometheus query error: %v", err)
+		return true
+	}
+	count := len(alertNames)
+	if count > 10 {
+		alertNames = append(alertNames[:10], "...")
+	}
+	if count > 0 {
+		log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
+		return true
+	}
+	return false
+}
+
+// MetricLabel is used to give a fancier name
+// than the type to the label for rebootBlockedCounter
+func (pb PrometheusBlockingChecker) MetricLabel() string {
+	return "prometheus"
+}
+
+// ActiveAlerts is a method of type promClient, it returns a list of names of active alerts
+// (e.g. pending or firing), filtered by the supplied regexp or by the includeLabels query.
+// filter by regexp means when the regexp finds the alert-name; the alert is excluded from the
+// block-list and will NOT block rebooting. query by includeLabel means,
+// if the query finds an alert, it will include it to the block-list, and it WILL block rebooting.
+func (pb PrometheusBlockingChecker) ActiveAlerts() ([]string, error) {
+	api := v1.NewAPI(pb.promClient)
+
+	// get all alerts from prometheus
+	value, _, err := api.Query(context.Background(), "ALERTS", time.Now())
+	if err != nil {
+		return nil, err
+	}
+
+	if value.Type() == model.ValVector {
+		if vector, ok := value.(model.Vector); ok {
+			activeAlertSet := make(map[string]bool)
+			for _, sample := range vector {
+				if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 {
+					if matchesRegex(pb.filter, string(alertName), pb.filterMatchOnly) && (!pb.firingOnly || sample.Metric["alertstate"] == "firing") {
+						activeAlertSet[string(alertName)] = true
+					}
+				}
+			}
+
+			var activeAlerts []string
+			for activeAlert := range activeAlertSet {
+				activeAlerts = append(activeAlerts, activeAlert)
+			}
+			sort.Strings(activeAlerts)
+
+			return activeAlerts, nil
+		}
+	}
+
+	return nil, fmt.Errorf("unexpected value type %v", value)
+}
+
+func matchesRegex(filter *regexp.Regexp, alertName string, filterMatchOnly bool) bool {
+	if filter == nil {
+		return true
+	}
+
+	return filter.MatchString(alertName) == filterMatchOnly
+}

pkg/blockers/prometheus_test.go

@@ -0,0 +1,163 @@
+package blockers
+
+import (
+	"log"
+	"net/http"
+	"net/http/httptest"
+
+	"regexp"
+	"testing"
+
+	"github.com/prometheus/client_golang/api"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type MockResponse struct {
+	StatusCode int
+	Body       []byte
+}
+
+// MockServerProperties ties a mock response to a url and a method
+type MockServerProperties struct {
+	URI        string
+	HTTPMethod string
+	Response   MockResponse
+}
+
+// NewMockServer sets up a new MockServer with properties ad starts the server.
+func NewMockServer(props ...MockServerProperties) *httptest.Server {
+
+	handler := http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			for _, proc := range props {
+				_, err := w.Write(proc.Response.Body)
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+		})
+	return httptest.NewServer(handler)
+}
+
+func TestActiveAlerts(t *testing.T) {
+	responsebody := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"GatekeeperViolations","alertstate":"firing","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"PodCrashing-dev","alertstate":"firing","container":"deployment","instance":"1.2.3.4:8080","job":"kube-state-metrics","namespace":"dev","pod":"dev-deployment-78dcbmf25v","severity":"critical","team":"dev"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"PodRestart-dev","alertstate":"firing","container":"deployment","instance":"1.2.3.4:1234","job":"kube-state-metrics","namespace":"qa","pod":"qa-job-deployment-78dcbmf25v","severity":"warning","team":"qa"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"PrometheusTargetDown","alertstate":"firing","job":"kubernetes-pods","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]},{"metric":{"__name__":"ALERTS","alertname":"ScheduledRebootFailing","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`
+	addr := "http://localhost:10001"
+
+	for _, tc := range []struct {
+		it              string
+		rFilter         string
+		respBody        string
+		aName           string
+		wantN           int
+		firingOnly      bool
+		filterMatchOnly bool
+	}{
+		{
+			it:              "should return no active alerts",
+			respBody:        responsebody,
+			rFilter:         "",
+			wantN:           0,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should return a subset of all alerts",
+			respBody:        responsebody,
+			rFilter:         "Pod",
+			wantN:           3,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should return a subset of all alerts",
+			respBody:        responsebody,
+			rFilter:         "Gatekeeper",
+			wantN:           1,
+			firingOnly:      false,
+			filterMatchOnly: true,
+		},
+		{
+			it:              "should return all active alerts by regex",
+			respBody:        responsebody,
+			rFilter:         "*",
+			wantN:           5,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should return all active alerts by regex filter",
+			respBody:        responsebody,
+			rFilter:         "*",
+			wantN:           5,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should return only firing alerts if firingOnly is true",
+			respBody:        responsebody,
+			rFilter:         "*",
+			wantN:           4,
+			firingOnly:      true,
+			filterMatchOnly: false,
+		},
+
+		{
+			it:              "should return ScheduledRebootFailing active alerts",
+			respBody:        `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"ScheduledRebootFailing","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`,
+			aName:           "ScheduledRebootFailing",
+			rFilter:         "*",
+			wantN:           1,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should not return an active alert if RebootRequired is firing (regex filter)",
+			respBody:        `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"RebootRequired","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`,
+			rFilter:         "RebootRequired",
+			wantN:           0,
+			firingOnly:      false,
+			filterMatchOnly: false,
+		},
+		{
+			it:              "should not return an active alert if RebootRequired is firing (regex filter)",
+			respBody:        `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"ALERTS","alertname":"RebootRequired","alertstate":"pending","severity":"warning","team":"platform-infra"},"value":[1622472933.973,"1"]}]}}`,
+			rFilter:         "RebootRequired",
+			wantN:           1,
+			firingOnly:      false,
+			filterMatchOnly: true,
+		},
+	} {
+		// Start mockServer
+		mockServer := NewMockServer(MockServerProperties{
+			URI:        addr,
+			HTTPMethod: http.MethodPost,
+			Response: MockResponse{
+				Body: []byte(tc.respBody),
+			},
+		})
+		// Close mockServer after all connections are gone
+		defer mockServer.Close()
+
+		t.Run(tc.it, func(t *testing.T) {
+
+			// regex filter
+			regex, _ := regexp.Compile(tc.rFilter)
+
+			// instantiate the prometheus client with the mockserver-address
+			p := NewPrometheusBlockingChecker(api.Config{Address: mockServer.URL}, regex, tc.firingOnly, tc.filterMatchOnly)
+
+			result, err := p.ActiveAlerts()
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			// assert
+			assert.Equal(t, tc.wantN, len(result), "expected amount of alerts %v, got %v", tc.wantN, len(result))
+
+			if tc.aName != "" {
+				assert.Equal(t, tc.aName, result[0], "expected active alert %v, got %v", tc.aName, result[0])
+			}
+		})
+	}
+}

pkg/checkers/checker.go

@@ -0,0 +1,117 @@
+// Package checkers provides interfaces and implementations for determining
+// whether a system reboot is required. It includes checkers based on file
+// presence or custom commands, and supports privileged command execution
+// in containerized environments. These checkers are used by kured to
+// detect conditions that should trigger node reboots.
+// You can use that package if you fork Kured's main loop.
+package checkers
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/google/shlex"
+	log "github.com/sirupsen/logrus"
+)
+
+// Checker is the standard interface to use to check
+// if a reboot is required. Its types must implement a
+// CheckRebootRequired method which returns a single boolean
+// clarifying whether a reboot is expected or not.
+type Checker interface {
+	RebootRequired() bool
+}
+
+// FileRebootChecker is the default reboot checker.
+// It is unprivileged, and tests the presence of a files
+type FileRebootChecker struct {
+	FilePath string
+}
+
+// RebootRequired checks the file presence
+// needs refactoring to also return an error, instead of leaking it inside the code.
+// This needs refactoring to get rid of NewCommand
+// This needs refactoring to only contain file location, instead of CheckCommand
+func (rc FileRebootChecker) RebootRequired() bool {
+	if _, err := os.Stat(rc.FilePath); err == nil {
+		return true
+	}
+	return false
+}
+
+// NewFileRebootChecker is the constructor for the file based reboot checker
+// TODO: Add extra input validation on filePath string here
+func NewFileRebootChecker(filePath string) (*FileRebootChecker, error) {
+	return &FileRebootChecker{
+		FilePath: filePath,
+	}, nil
+}
+
+// CommandChecker is using a custom command to check
+// if a reboot is required. There are two modes of behaviour,
+// if Privileged is granted, the NamespacePid is used to nsenter
+// the given PID's namespace.
+type CommandChecker struct {
+	CheckCommand []string
+	NamespacePid int
+	Privileged   bool
+}
+
+// RebootRequired for CommandChecker runs a command without returning
+// any eventual error. This should be later refactored to return the errors,
+// instead of logging and fataling them here.
+func (rc CommandChecker) RebootRequired() bool {
+	bufStdout := new(bytes.Buffer)
+	bufStderr := new(bytes.Buffer)
+	// #nosec G204 -- CheckCommand is controlled and validated internally
+	cmd := exec.Command(rc.CheckCommand[0], rc.CheckCommand[1:]...)
+	cmd.Stdout = bufStdout
+	cmd.Stderr = bufStderr
+
+	if err := cmd.Run(); err != nil {
+		switch err := err.(type) {
+		case *exec.ExitError:
+			// We assume a non-zero exit code means 'reboot not required', but of course
+			// the user could have misconfigured the sentinel command or something else
+			// went wrong during its execution. In that case, not entering a reboot loop
+			// is the right thing to do, and we are logging stdout/stderr of the command
+			// so it should be obvious what is wrong.
+			if cmd.ProcessState.ExitCode() != 1 {
+				log.Warn(fmt.Sprintf("sentinel command ended with unexpected exit code: %v", cmd.ProcessState.ExitCode()), "cmd", strings.Join(cmd.Args, " "), "stdout", bufStdout.String(), "stderr", bufStderr.String())
+			}
+			return false
+		default:
+			// Something was grossly misconfigured, such as the command path being wrong.
+			log.Fatal(fmt.Sprintf("Error invoking sentinel command: %v", err), "cmd", strings.Join(cmd.Args, " "), "stdout", bufStdout.String(), "stderr", bufStderr.String())
+		}
+	}
+	log.Info("checking if reboot is required", "cmd", strings.Join(cmd.Args, " "), "stdout", bufStdout.String(), "stderr", bufStderr.String())
+	return true
+}
+
+// NewCommandChecker is the constructor for the commandChecker, and by default
+// runs new commands in a privileged fashion.
+// Privileged means wrapping the command with nsenter.
+// It allows to run a command from systemd's namespace for example (pid 1)
+// This relies on hostPID:true and privileged:true to enter host mount space
+// For info, rancher based need different pid, which should be user given.
+// until we have a better discovery mechanism.
+func NewCommandChecker(sentinelCommand string, pid int, privileged bool) (*CommandChecker, error) {
+	var cmd []string
+	if privileged {
+		cmd = append(cmd, "/usr/bin/nsenter", fmt.Sprintf("-m/proc/%d/ns/mnt", pid), "--")
+	}
+	parsedCommand, err := shlex.Split(sentinelCommand)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing provided sentinel command: %v", err)
+	}
+	cmd = append(cmd, parsedCommand...)
+	return &CommandChecker{
+		CheckCommand: cmd,
+		NamespacePid: pid,
+		Privileged:   privileged,
+	}, nil
+}

pkg/checkers/checker_test.go

@@ -0,0 +1,87 @@
+package checkers
+
+import (
+	log "github.com/sirupsen/logrus"
+	"reflect"
+	"testing"
+)
+
+func Test_nsEntering(t *testing.T) {
+	type args struct {
+		pid        int
+		command    string
+		privileged bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "Ensure command will run with nsenter",
+			args: args{pid: 1, command: "ls -Fal", privileged: true},
+			want: []string{"/usr/bin/nsenter", "-m/proc/1/ns/mnt", "--", "ls", "-Fal"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cc, _ := NewCommandChecker(tt.args.command, tt.args.pid, tt.args.privileged)
+			if !reflect.DeepEqual(cc.CheckCommand, tt.want) {
+				t.Errorf("command parsed as %v, want %v", cc.CheckCommand, tt.want)
+			}
+		})
+	}
+}
+
+func Test_rebootRequired(t *testing.T) {
+	type args struct {
+		sentinelCommand []string
+	}
+	tests := []struct {
+		name   string
+		args   args
+		want   bool
+		fatals bool
+	}{
+		{
+			name: "Ensure rc = 0 means reboot required",
+			args: args{
+				sentinelCommand: []string{"true"},
+			},
+			want:   true,
+			fatals: false,
+		},
+		{
+			name: "Ensure rc != 0 means reboot NOT required",
+			args: args{
+				sentinelCommand: []string{"false"},
+			},
+			want:   false,
+			fatals: false,
+		},
+		{
+			name: "Ensure a wrong command fatals",
+			args: args{
+				sentinelCommand: []string{"./babar"},
+			},
+			want:   true,
+			fatals: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			defer func() { log.StandardLogger().ExitFunc = nil }()
+			fatal := false
+			log.StandardLogger().ExitFunc = func(int) { fatal = true }
+
+			a := CommandChecker{CheckCommand: tt.args.sentinelCommand, NamespacePid: 1, Privileged: false}
+
+			if got := a.RebootRequired(); got != tt.want {
+				t.Errorf("rebootRequired() = %v, want %v", got, tt.want)
+			}
+			if tt.fatals != fatal {
+				t.Errorf("fatal flag is %v, want fatal %v", fatal, tt.fatals)
+			}
+		})
+	}
+}

pkg/daemonsetlock/daemonsetlock.go

@@ -1,15 +1,51 @@
+// Package daemonsetlock provides mechanisms for leader election and locking
+// using Kubernetes DaemonSets. It enables distributed coordination of operations
+// (such as reboots) by ensuring only one node acts as the leader at any time,
+// leveraging Kubernetes primitives for safe, atomic locking in clusters.
 package daemonsetlock
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
+	v1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 )
 
+const (
+	k8sAPICallRetrySleep   = 5 * time.Second // How much time to wait in between retrying a k8s API call
+	k8sAPICallRetryTimeout = 5 * time.Minute // How long to wait until we determine that the k8s API is definitively unavailable
+)
+
+// Lock defines the interface for acquiring, releasing, and checking
+// the status of a reboot coordination lock.
+type Lock interface {
+	Acquire(NodeMeta) (bool, string, error)
+	Release() error
+	Holding() (bool, LockAnnotationValue, error)
+}
+
+// GenericLock holds the configuration for lock TTL and the delay before releasing it.
+type GenericLock struct {
+	TTL          time.Duration
+	releaseDelay time.Duration
+}
+
+// NodeMeta contains metadata about a node relevant to scheduling decisions.
+type NodeMeta struct {
+	Unschedulable bool `json:"unschedulable"`
+}
+
+// DaemonSetLock holds all necessary information to do actions
+// on the kured ds which holds lock info through annotations.
 type DaemonSetLock struct {
 	client     *kubernetes.Clientset
 	nodeID     string
@@ -18,104 +54,367 @@ type DaemonSetLock struct {
 	annotation string
 }
 
-type lockAnnotationValue struct {
-	NodeID   string      `json:"nodeID"`
-	Metadata interface{} `json:"metadata,omitempty"`
+// DaemonSetSingleLock holds all necessary information to do actions
+// on the kured ds which holds lock info through annotations.
+type DaemonSetSingleLock struct {
+	GenericLock
+	DaemonSetLock
 }
 
-func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation string) *DaemonSetLock {
-	return &DaemonSetLock{client, nodeID, namespace, name, annotation}
+// DaemonSetMultiLock holds all necessary information to do actions
+// on the kured ds which holds lock info through annotations, valid
+// for multiple nodes
+type DaemonSetMultiLock struct {
+	GenericLock
+	DaemonSetLock
+	maxOwners int
 }
 
-func (dsl *DaemonSetLock) Acquire(metadata interface{}) (acquired bool, owner string, err error) {
+// LockAnnotationValue contains the lock data,
+// which allows persistence across reboots, particularily recording if the
+// node was already unschedulable before kured reboot.
+// To be modified when using another type of lock storage.
+type LockAnnotationValue struct {
+	NodeID   string        `json:"nodeID"`
+	Metadata NodeMeta      `json:"metadata,omitempty"`
+	Created  time.Time     `json:"created"`
+	TTL      time.Duration `json:"TTL"`
+}
+
+type multiLockAnnotationValue struct {
+	MaxOwners       int                   `json:"maxOwners"`
+	LockAnnotations []LockAnnotationValue `json:"locks"`
+}
+
+// New creates a daemonsetLock object containing the necessary data for follow up k8s requests
+func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation string, TTL time.Duration, concurrency int, lockReleaseDelay time.Duration) Lock {
+	if concurrency > 1 {
+		return &DaemonSetMultiLock{
+			GenericLock: GenericLock{
+				TTL:          TTL,
+				releaseDelay: lockReleaseDelay,
+			},
+			DaemonSetLock: DaemonSetLock{
+				client:     client,
+				nodeID:     nodeID,
+				namespace:  namespace,
+				name:       name,
+				annotation: annotation,
+			},
+			maxOwners: concurrency,
+		}
+	}
+	return &DaemonSetSingleLock{
+		GenericLock: GenericLock{
+			TTL:          TTL,
+			releaseDelay: lockReleaseDelay,
+		},
+		DaemonSetLock: DaemonSetLock{
+			client:     client,
+			nodeID:     nodeID,
+			namespace:  namespace,
+			name:       name,
+			annotation: annotation,
+		},
+	}
+}
+
+// GetDaemonSet returns the named DaemonSet resource from the DaemonSetLock's configured client
+func (dsl *DaemonSetLock) GetDaemonSet(sleep, timeout time.Duration) (*v1.DaemonSet, error) {
+	var ds *v1.DaemonSet
+	var lastError error
+	err := wait.PollUntilContextTimeout(context.Background(), sleep, timeout, true, func(ctx context.Context) (bool, error) {
+		if ds, lastError = dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(ctx, dsl.name, metav1.GetOptions{}); lastError != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %v", dsl.name, dsl.namespace, lastError)
+	}
+	return ds, nil
+}
+
+// Acquire attempts to annotate the kured daemonset with lock info from instantiated DaemonSetLock using client-go
+func (dsl *DaemonSetSingleLock) Acquire(nodeMetadata NodeMeta) (bool, string, error) {
 	for {
-		ds, err := dsl.client.ExtensionsV1beta1().DaemonSets(dsl.namespace).Get(dsl.name, metav1.GetOptions{})
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
-			return false, "", err
+			return false, "", fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 		}
 
-		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
+		valueString, exists := ds.Annotations[dsl.annotation]
 		if exists {
-			value := lockAnnotationValue{}
+			value := LockAnnotationValue{}
 			if err := json.Unmarshal([]byte(valueString), &value); err != nil {
 				return false, "", err
 			}
-			return value.NodeID == dsl.nodeID, value.NodeID, nil
+
+			if !ttlExpired(value.Created, value.TTL) {
+				return value.NodeID == dsl.nodeID, value.NodeID, nil
+			}
 		}
 
-		if ds.ObjectMeta.Annotations == nil {
-			ds.ObjectMeta.Annotations = make(map[string]string)
+		if ds.Annotations == nil {
+			ds.Annotations = make(map[string]string)
 		}
-		value := lockAnnotationValue{NodeID: dsl.nodeID, Metadata: metadata}
+
+		value := LockAnnotationValue{
+			NodeID:   dsl.nodeID,
+			Metadata: nodeMetadata,
+			Created:  time.Now().UTC(),
+			TTL:      dsl.TTL,
+		}
+
 		valueBytes, err := json.Marshal(&value)
 		if err != nil {
 			return false, "", err
 		}
-		ds.ObjectMeta.Annotations[dsl.annotation] = string(valueBytes)
+		ds.Annotations[dsl.annotation] = string(valueBytes)
 
-		_, err = dsl.client.ExtensionsV1beta1().DaemonSets(dsl.namespace).Update(ds)
+		_, err = dsl.client.AppsV1().DaemonSets(dsl.namespace).Update(context.TODO(), ds, metav1.UpdateOptions{})
 		if err != nil {
 			if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == metav1.StatusReasonConflict {
 				// Something else updated the resource between us reading and writing - try again soon
 				time.Sleep(time.Second)
 				continue
-			} else {
-				return false, "", err
 			}
+			return false, "", err
 		}
+
 		return true, dsl.nodeID, nil
 	}
 }
 
-func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) {
-	ds, err := dsl.client.ExtensionsV1beta1().DaemonSets(dsl.namespace).Get(dsl.name, metav1.GetOptions{})
+// Holding checks if the current node still holds the lock based on the DaemonSet annotation.
+func (dsl *DaemonSetSingleLock) Holding() (bool, LockAnnotationValue, error) {
+	var lockData LockAnnotationValue
+	ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 	if err != nil {
-		return false, err
+		return false, lockData, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 	}
 
-	valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
+	valueString, exists := ds.Annotations[dsl.annotation]
 	if exists {
-		value := lockAnnotationValue{Metadata: metadata}
+		value := LockAnnotationValue{}
 		if err := json.Unmarshal([]byte(valueString), &value); err != nil {
-			return false, err
+			return false, lockData, err
+		}
+
+		if !ttlExpired(value.Created, value.TTL) {
+			return value.NodeID == dsl.nodeID, value, nil
 		}
-		return value.NodeID == dsl.nodeID, nil
 	}
 
-	return false, nil
+	return false, lockData, nil
 }
 
-func (dsl *DaemonSetLock) Release() error {
+// Release attempts to remove the lock data from the kured ds annotations using client-go
+func (dsl *DaemonSetSingleLock) Release() error {
+	if dsl.releaseDelay > 0 {
+		log.Infof("Waiting %v before releasing lock", dsl.releaseDelay)
+		time.Sleep(dsl.releaseDelay)
+	}
 	for {
-		ds, err := dsl.client.ExtensionsV1beta1().DaemonSets(dsl.namespace).Get(dsl.name, metav1.GetOptions{})
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
-			return err
+			return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 		}
 
-		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
+		valueString, exists := ds.Annotations[dsl.annotation]
 		if exists {
-			value := lockAnnotationValue{}
+			value := LockAnnotationValue{}
 			if err := json.Unmarshal([]byte(valueString), &value); err != nil {
 				return err
 			}
+
 			if value.NodeID != dsl.nodeID {
-				return fmt.Errorf("Not lock holder: %v", value.NodeID)
+				return fmt.Errorf("not lock holder: %v", value.NodeID)
 			}
 		} else {
-			return fmt.Errorf("Lock not held")
+			return fmt.Errorf("lock not held")
 		}
 
-		delete(ds.ObjectMeta.Annotations, dsl.annotation)
+		delete(ds.Annotations, dsl.annotation)
 
-		_, err = dsl.client.ExtensionsV1beta1().DaemonSets(dsl.namespace).Update(ds)
+		_, err = dsl.client.AppsV1().DaemonSets(dsl.namespace).Update(context.TODO(), ds, metav1.UpdateOptions{})
 		if err != nil {
 			if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == metav1.StatusReasonConflict {
 				// Something else updated the resource between us reading and writing - try again soon
 				time.Sleep(time.Second)
 				continue
-			} else {
-				return err
 			}
+			return err
+		}
+		return nil
+	}
+}
+
+func ttlExpired(created time.Time, ttl time.Duration) bool {
+	if ttl > 0 && time.Since(created) >= ttl {
+		return true
+	}
+	return false
+}
+
+func nodeIDsFromMultiLock(annotation multiLockAnnotationValue) []string {
+	nodeIDs := make([]string, 0, len(annotation.LockAnnotations))
+	for _, nodeLock := range annotation.LockAnnotations {
+		nodeIDs = append(nodeIDs, nodeLock.NodeID)
+	}
+	return nodeIDs
+}
+
+func (dsl *DaemonSetLock) canAcquireMultiple(annotation multiLockAnnotationValue, metadata NodeMeta, TTL time.Duration, maxOwners int) (bool, multiLockAnnotationValue) {
+	newAnnotation := multiLockAnnotationValue{MaxOwners: maxOwners}
+	freeSpace := false
+	if annotation.LockAnnotations == nil || len(annotation.LockAnnotations) < maxOwners {
+		freeSpace = true
+		newAnnotation.LockAnnotations = annotation.LockAnnotations
+	} else {
+		for _, nodeLock := range annotation.LockAnnotations {
+			if ttlExpired(nodeLock.Created, nodeLock.TTL) {
+				freeSpace = true
+				continue
+			}
+			newAnnotation.LockAnnotations = append(
+				newAnnotation.LockAnnotations,
+				nodeLock,
+			)
+		}
+	}
+
+	if freeSpace {
+		newAnnotation.LockAnnotations = append(
+			newAnnotation.LockAnnotations,
+			LockAnnotationValue{
+				NodeID:   dsl.nodeID,
+				Metadata: metadata,
+				Created:  time.Now().UTC(),
+				TTL:      TTL,
+			},
+		)
+		return true, newAnnotation
+	}
+
+	return false, multiLockAnnotationValue{}
+}
+
+// Acquire creates and annotates the daemonset with a multiple owner lock
+func (dsl *DaemonSetMultiLock) Acquire(nodeMetaData NodeMeta) (bool, string, error) {
+	for {
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
+		if err != nil {
+			return false, "", fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
+		}
+
+		annotation := multiLockAnnotationValue{}
+		valueString, exists := ds.Annotations[dsl.annotation]
+		if exists {
+			if err := json.Unmarshal([]byte(valueString), &annotation); err != nil {
+				return false, "", fmt.Errorf("error getting multi lock: %w", err)
+			}
+		}
+
+		lockPossible, newAnnotation := dsl.canAcquireMultiple(annotation, nodeMetaData, dsl.TTL, dsl.maxOwners)
+		if !lockPossible {
+			return false, strings.Join(nodeIDsFromMultiLock(newAnnotation), ","), nil
+		}
+
+		if ds.Annotations == nil {
+			ds.Annotations = make(map[string]string)
+		}
+		newAnnotationBytes, err := json.Marshal(&newAnnotation)
+		if err != nil {
+			return false, "", fmt.Errorf("error marshalling new annotation lock: %w", err)
+		}
+		ds.Annotations[dsl.annotation] = string(newAnnotationBytes)
+
+		_, err = dsl.client.AppsV1().DaemonSets(dsl.namespace).Update(context.Background(), ds, metav1.UpdateOptions{})
+		if err != nil {
+			if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == metav1.StatusReasonConflict {
+				time.Sleep(time.Second)
+				continue
+			}
+			return false, "", fmt.Errorf("error updating daemonset with multi lock: %w", err)
+
+		}
+		return true, strings.Join(nodeIDsFromMultiLock(newAnnotation), ","), nil
+	}
+}
+
+// Holding checks whether the current node is holding a valid lock for the DaemonSetMultiLock.
+func (dsl *DaemonSetMultiLock) Holding() (bool, LockAnnotationValue, error) {
+	var lockdata LockAnnotationValue
+	ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
+	if err != nil {
+		return false, lockdata, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
+	}
+
+	valueString, exists := ds.Annotations[dsl.annotation]
+	if exists {
+		value := multiLockAnnotationValue{}
+		if err := json.Unmarshal([]byte(valueString), &value); err != nil {
+			return false, lockdata, err
+		}
+
+		for _, nodeLock := range value.LockAnnotations {
+			if nodeLock.NodeID == dsl.nodeID && !ttlExpired(nodeLock.Created, nodeLock.TTL) {
+				return true, nodeLock, nil
+			}
+		}
+	}
+
+	return false, lockdata, nil
+}
+
+// Release attempts to remove the lock data for a single node from the multi node annotation
+func (dsl *DaemonSetMultiLock) Release() error {
+	if dsl.releaseDelay > 0 {
+		log.Infof("Waiting %v before releasing lock", dsl.releaseDelay)
+		time.Sleep(dsl.releaseDelay)
+	}
+	for {
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
+		if err != nil {
+			return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
+		}
+
+		valueString, exists := ds.Annotations[dsl.annotation]
+		modified := false
+		value := multiLockAnnotationValue{}
+		if exists {
+			if err := json.Unmarshal([]byte(valueString), &value); err != nil {
+				return err
+			}
+
+			for idx, nodeLock := range value.LockAnnotations {
+				if nodeLock.NodeID == dsl.nodeID {
+					value.LockAnnotations = append(value.LockAnnotations[:idx], value.LockAnnotations[idx+1:]...)
+					modified = true
+					break
+				}
+			}
+		}
+
+		if !exists || !modified {
+			return fmt.Errorf("Lock not held")
+		}
+
+		newAnnotationBytes, err := json.Marshal(value)
+		if err != nil {
+			return fmt.Errorf("error marshalling new annotation on release: %v", err)
+		}
+		ds.Annotations[dsl.annotation] = string(newAnnotationBytes)
+
+		_, err = dsl.client.AppsV1().DaemonSets(dsl.namespace).Update(context.TODO(), ds, metav1.UpdateOptions{})
+		if err != nil {
+			if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == metav1.StatusReasonConflict {
+				// Something else updated the resource between us reading and writing - try again soon
+				time.Sleep(time.Second)
+				continue
+			}
+			return err
 		}
 		return nil
 	}

pkg/daemonsetlock/daemonsetlock_test.go

@@ -0,0 +1,208 @@
+package daemonsetlock
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+)
+
+func TestTtlExpired(t *testing.T) {
+	d := time.Date(2020, 05, 05, 14, 15, 0, 0, time.UTC)
+	second, _ := time.ParseDuration("1s")
+	zero, _ := time.ParseDuration("0m")
+
+	tests := []struct {
+		created time.Time
+		ttl     time.Duration
+		result  bool
+	}{
+		{d, second, true},
+		{time.Now(), second, false},
+		{d, zero, false},
+	}
+
+	for i, tst := range tests {
+		if ttlExpired(tst.created, tst.ttl) != tst.result {
+			t.Errorf("Test %d failed, expected %v but got %v", i, tst.result, !tst.result)
+		}
+	}
+}
+
+func multiLockAnnotationsAreEqualByNodes(src, dst multiLockAnnotationValue) bool {
+	srcNodes := []string{}
+	for _, srcLock := range src.LockAnnotations {
+		srcNodes = append(srcNodes, srcLock.NodeID)
+	}
+	sort.Strings(srcNodes)
+
+	dstNodes := []string{}
+	for _, dstLock := range dst.LockAnnotations {
+		dstNodes = append(dstNodes, dstLock.NodeID)
+	}
+	sort.Strings(dstNodes)
+
+	return reflect.DeepEqual(srcNodes, dstNodes)
+}
+
+func TestCanAcquireMultiple(t *testing.T) {
+	node1Name := "n1"
+	node2Name := "n2"
+	node3Name := "n3"
+	testCases := []struct {
+		name          string
+		daemonSetLock DaemonSetLock
+		maxOwners     int
+		current       multiLockAnnotationValue
+		desired       multiLockAnnotationValue
+		lockPossible  bool
+	}{
+		{
+			name: "empty_lock",
+			daemonSetLock: DaemonSetLock{
+				nodeID: node1Name,
+			},
+			maxOwners: 2,
+			current:   multiLockAnnotationValue{},
+			desired: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []LockAnnotationValue{
+					{NodeID: node1Name},
+				},
+			},
+			lockPossible: true,
+		},
+		{
+			name: "partial_lock",
+			daemonSetLock: DaemonSetLock{
+				nodeID: node1Name,
+			},
+			maxOwners: 2,
+			current: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []LockAnnotationValue{
+					{NodeID: node2Name},
+				},
+			},
+			desired: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []LockAnnotationValue{
+					{NodeID: node1Name},
+					{NodeID: node2Name},
+				},
+			},
+			lockPossible: true,
+		},
+		{
+			name: "full_lock",
+			daemonSetLock: DaemonSetLock{
+				nodeID: node1Name,
+			},
+			maxOwners: 2,
+			current: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []LockAnnotationValue{
+					{
+						NodeID:  node2Name,
+						Created: time.Now().UTC().Add(-1 * time.Minute),
+						TTL:     time.Hour,
+					},
+					{
+						NodeID:  node3Name,
+						Created: time.Now().UTC().Add(-1 * time.Minute),
+						TTL:     time.Hour,
+					},
+				},
+			},
+			desired: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []LockAnnotationValue{
+					{NodeID: node2Name},
+					{NodeID: node3Name},
+				},
+			},
+			lockPossible: false,
+		},
+		{
+			name: "full_with_one_expired_lock",
+			daemonSetLock: DaemonSetLock{
+				nodeID: node1Name,
+			},
+			maxOwners: 2,
+			current: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []LockAnnotationValue{
+					{
+						NodeID:  node2Name,
+						Created: time.Now().UTC().Add(-1 * time.Hour),
+						TTL:     time.Minute,
+					},
+					{
+						NodeID:  node3Name,
+						Created: time.Now().UTC().Add(-1 * time.Minute),
+						TTL:     time.Hour,
+					},
+				},
+			},
+			desired: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []LockAnnotationValue{
+					{NodeID: node1Name},
+					{NodeID: node3Name},
+				},
+			},
+			lockPossible: true,
+		},
+		{
+			name: "full_with_all_expired_locks",
+			daemonSetLock: DaemonSetLock{
+				nodeID: node1Name,
+			},
+			maxOwners: 2,
+			current: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []LockAnnotationValue{
+					{
+						NodeID:  node2Name,
+						Created: time.Now().UTC().Add(-1 * time.Hour),
+						TTL:     time.Minute,
+					},
+					{
+						NodeID:  node3Name,
+						Created: time.Now().UTC().Add(-1 * time.Hour),
+						TTL:     time.Minute,
+					},
+				},
+			},
+			desired: multiLockAnnotationValue{
+				MaxOwners: 2,
+				LockAnnotations: []LockAnnotationValue{
+					{NodeID: node1Name},
+				},
+			},
+			lockPossible: true,
+		},
+	}
+	nm := NodeMeta{Unschedulable: false}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			lockPossible, actual := testCase.daemonSetLock.canAcquireMultiple(testCase.current, nm, time.Minute, testCase.maxOwners)
+			if lockPossible != testCase.lockPossible {
+				t.Fatalf(
+					"unexpected result for lock possible (got %t expected %t new annotation %v",
+					lockPossible,
+					testCase.lockPossible,
+					actual,
+				)
+			}
+
+			if lockPossible && (!multiLockAnnotationsAreEqualByNodes(actual, testCase.desired) || testCase.desired.MaxOwners != actual.MaxOwners) {
+				t.Fatalf(
+					"expected lock %v but got %v",
+					testCase.desired,
+					actual,
+				)
+			}
+		})
+	}
+}

pkg/delaytick/delaytick.go

@@ -1,3 +1,9 @@
+// Package delaytick provides utilities for scheduling periodic events
+// with an initial randomized delay. It is primarily used to delay the
+// start of regular ticks, helping to avoid thundering herd problems
+// when multiple nodes begin operations simultaneously.
+// You can use that a random ticker in other projects, but there is
+// no garantee that it will stay (initial plan was to move it to internal)
 package delaytick
 
 import (
@@ -5,11 +11,12 @@ import (
 	"time"
 )
 
-// Tick regularly after an initial delay randomly distributed between d/2 and d + d/2
+// New ticks regularly after an initial delay randomly distributed between d/2 and d + d/2
 func New(s rand.Source, d time.Duration) <-chan time.Time {
 	c := make(chan time.Time)
 
 	go func() {
+		// #nosec G404 -- math/rand is used here for non-security timing jitter
 		random := rand.New(s)
 		time.Sleep(time.Duration(float64(d)/2 + float64(d)*random.Float64()))
 		c <- time.Now()

pkg/notifications/slack/slack.go

@@ -1,42 +0,0 @@
-package slack
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"time"
-)
-
-var (
-	httpClient = &http.Client{Timeout: 5 * time.Second}
-)
-
-type body struct {
-	Text     string `json:"text,omitempty"`
-	Username string `json:"username,omitempty"`
-}
-
-func NotifyReboot(hookURL, username, nodeID string) error {
-	msg := body{
-		Text:     fmt.Sprintf("Rebooting node %s", nodeID),
-		Username: username,
-	}
-
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(&msg); err != nil {
-		return err
-	}
-
-	resp, err := httpClient.Post(hookURL, "application/json", &buf)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf(resp.Status)
-	}
-
-	return nil
-}

pkg/reboot/command.go

@@ -0,0 +1,49 @@
+package reboot
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/google/shlex"
+	log "github.com/sirupsen/logrus"
+)
+
+// CommandRebooter holds context-information for a reboot with command
+type CommandRebooter struct {
+	RebootCommand []string
+}
+
+// Reboot triggers the reboot command
+func (c CommandRebooter) Reboot() error {
+	log.Infof("Invoking command: %s", c.RebootCommand)
+
+	bufStdout := new(bytes.Buffer)
+	bufStderr := new(bytes.Buffer)
+	cmd := exec.Command(c.RebootCommand[0], c.RebootCommand[1:]...) // #nosec G204
+	cmd.Stdout = bufStdout
+	cmd.Stderr = bufStderr
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("error invoking reboot command %s: %v (stdout: %v, stderr: %v)", c.RebootCommand, err, bufStdout.String(), bufStderr.String())
+	}
+	log.Info("Invoked reboot command", "cmd", strings.Join(cmd.Args, " "), "stdout", bufStdout.String(), "stderr", bufStderr.String())
+	return nil
+}
+
+// NewCommandRebooter is the constructor to create a CommandRebooter from a string not
+// yet shell lexed. You can skip this constructor if you parse the data correctly first
+// when instantiating a CommandRebooter instance.
+func NewCommandRebooter(rebootCommand string) (*CommandRebooter, error) {
+	if rebootCommand == "" {
+		return nil, fmt.Errorf("no reboot command specified")
+	}
+	cmd := []string{"/usr/bin/nsenter", fmt.Sprintf("-m/proc/%d/ns/mnt", 1), "--"}
+	parsedCommand, err := shlex.Split(rebootCommand)
+	if err != nil {
+		return nil, fmt.Errorf("error %v when parsing reboot command %s", err, rebootCommand)
+	}
+	cmd = append(cmd, parsedCommand...)
+	return &CommandRebooter{RebootCommand: cmd}, nil
+}

pkg/reboot/command_test.go

@@ -0,0 +1,43 @@
+package reboot
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestNewCommandRebooter(t *testing.T) {
+	type args struct {
+		rebootCommand string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *CommandRebooter
+		wantErr bool
+	}{
+		{
+			name:    "Ensure command is nsenter wrapped",
+			args:    args{"ls -Fal"},
+			want:    &CommandRebooter{RebootCommand: []string{"/usr/bin/nsenter", "-m/proc/1/ns/mnt", "--", "ls", "-Fal"}},
+			wantErr: false,
+		},
+		{
+			name:    "Ensure empty command is erroring",
+			args:    args{""},
+			want:    nil,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := NewCommandRebooter(tt.args.rebootCommand)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("NewCommandRebooter() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("NewCommandRebooter() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/reboot/reboot.go

@@ -0,0 +1,13 @@
+// Package reboot provides mechanisms to trigger node reboots using different
+// methods, like custom commands or signals.
+// Each of those includes constructors and interfaces for handling different reboot
+// strategies, supporting privileged command execution via nsenter for containerized environments.
+package reboot
+
+// Rebooter is the standard interface to use to execute
+// the reboot, after it has been considered as necessary.
+// The Reboot method does not expect any return, yet should
+// most likely be refactored in the future to return an error
+type Rebooter interface {
+	Reboot() error
+}

pkg/reboot/signal.go

@@ -0,0 +1,37 @@
+package reboot
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+)
+
+// SignalRebooter holds context-information for a signal reboot.
+type SignalRebooter struct {
+	Signal int
+}
+
+// Reboot triggers the reboot signal
+func (c SignalRebooter) Reboot() error {
+	process, err := os.FindProcess(1)
+	if err != nil {
+		return fmt.Errorf("not running on Unix: %v", err)
+	}
+
+	err = process.Signal(syscall.Signal(c.Signal))
+	// Either PID does not exist, or the signal does not work. Hoping for
+	// a decent enough error.
+	if err != nil {
+		return fmt.Errorf("signal of SIGRTMIN+5 failed: %v", err)
+	}
+	return nil
+}
+
+// NewSignalRebooter is the constructor which sets the signal number.
+// The constructor does not yet validate any input. It should be done in a later commit.
+func NewSignalRebooter(sig int) (*SignalRebooter, error) {
+	if sig < 1 {
+		return nil, fmt.Errorf("invalid signal: %v", sig)
+	}
+	return &SignalRebooter{Signal: sig}, nil
+}

pkg/taints/taints.go

@@ -0,0 +1,169 @@
+// Package taints provides utilities to manage Kubernetes node taints for controlling
+// pod scheduling and execution. It allows setting, removing, and checking taints on nodes,
+// using Kubernetes client-go and JSON patching for atomic updates.
+package taints
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Taint allows to set soft and hard limitations for scheduling and executing pods on nodes.
+type Taint struct {
+	client    *kubernetes.Clientset
+	nodeID    string
+	taintName string
+	effect    v1.TaintEffect
+	exists    bool
+}
+
+// New provides a new taint.
+func New(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect) *Taint {
+	exists, _, _ := taintExists(client, nodeID, taintName)
+
+	return &Taint{
+		client:    client,
+		nodeID:    nodeID,
+		taintName: taintName,
+		effect:    effect,
+		exists:    exists,
+	}
+}
+
+// Enable creates the taint for a node. Creating an existing taint is a noop.
+func (t *Taint) Enable() {
+	if t.taintName == "" {
+		return
+	}
+
+	if t.exists {
+		return
+	}
+
+	preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, true)
+
+	t.exists = true
+}
+
+// Disable removes the taint for a node. Removing a missing taint is a noop.
+func (t *Taint) Disable() {
+	if t.taintName == "" {
+		return
+	}
+
+	if !t.exists {
+		return
+	}
+
+	preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, false)
+
+	t.exists = false
+}
+
+func taintExists(client *kubernetes.Clientset, nodeID, taintName string) (bool, int, *v1.Node) {
+	updatedNode, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+	if err != nil || updatedNode == nil {
+		log.Fatalf("Error reading node %s: %v", nodeID, err)
+	}
+
+	for i, taint := range updatedNode.Spec.Taints {
+		if taint.Key == taintName {
+			return true, i, updatedNode
+		}
+	}
+
+	return false, 0, updatedNode
+}
+
+func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect, shouldExists bool) {
+	taintExists, offset, updatedNode := taintExists(client, nodeID, taintName)
+
+	if taintExists && shouldExists {
+		log.Debugf("Taint %v exists already for node %v.", taintName, nodeID)
+		return
+	}
+
+	if !taintExists && !shouldExists {
+		log.Debugf("Taint %v already missing for node %v.", taintName, nodeID)
+		return
+	}
+
+	type patchTaints struct {
+		Op    string      `json:"op"`
+		Path  string      `json:"path"`
+		Value interface{} `json:"value,omitempty"`
+	}
+
+	taint := v1.Taint{
+		Key:    taintName,
+		Effect: effect,
+	}
+
+	var patches []patchTaints
+
+	if len(updatedNode.Spec.Taints) == 0 {
+		// add first taint and ensure to keep current taints
+		patches = []patchTaints{
+			{
+				Op:    "test",
+				Path:  "/spec",
+				Value: updatedNode.Spec,
+			},
+			{
+				Op:    "add",
+				Path:  "/spec/taints",
+				Value: []v1.Taint{},
+			},
+			{
+				Op:    "add",
+				Path:  "/spec/taints/-",
+				Value: taint,
+			},
+		}
+	} else if taintExists {
+		// remove taint and ensure to test against race conditions
+		patches = []patchTaints{
+			{
+				Op:    "test",
+				Path:  fmt.Sprintf("/spec/taints/%d", offset),
+				Value: taint,
+			},
+			{
+				Op:   "remove",
+				Path: fmt.Sprintf("/spec/taints/%d", offset),
+			},
+		}
+	} else {
+		// add missing taint to exsting list
+		patches = []patchTaints{
+			{
+				Op:    "add",
+				Path:  "/spec/taints/-",
+				Value: taint,
+			},
+		}
+	}
+
+	patchBytes, err := json.Marshal(patches)
+	if err != nil {
+		log.Fatalf("Error encoding taint patch for node %s: %v", nodeID, err)
+	}
+
+	_, err = client.CoreV1().Nodes().Patch(context.TODO(), nodeID, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
+	if err != nil {
+		log.Fatalf("Error patching taint for node %s: %v", nodeID, err)
+	}
+
+	if shouldExists {
+		log.Info("Node taint added")
+	} else {
+		log.Info("Node taint removed")
+	}
+}

pkg/timewindow/days.go

@@ -0,0 +1,92 @@
+package timewindow
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// EveryDay contains all days of the week, and exports it
+// for convenience use in the cmd line arguments.
+var EveryDay = []string{"su", "mo", "tu", "we", "th", "fr", "sa"}
+
+// dayStrings maps day strings to time.Weekdays
+var dayStrings = map[string]time.Weekday{
+	"su":        time.Sunday,
+	"sun":       time.Sunday,
+	"sunday":    time.Sunday,
+	"mo":        time.Monday,
+	"mon":       time.Monday,
+	"monday":    time.Monday,
+	"tu":        time.Tuesday,
+	"tue":       time.Tuesday,
+	"tuesday":   time.Tuesday,
+	"we":        time.Wednesday,
+	"wed":       time.Wednesday,
+	"wednesday": time.Wednesday,
+	"th":        time.Thursday,
+	"thu":       time.Thursday,
+	"thursday":  time.Thursday,
+	"fr":        time.Friday,
+	"fri":       time.Friday,
+	"friday":    time.Friday,
+	"sa":        time.Saturday,
+	"sat":       time.Saturday,
+	"saturday":  time.Saturday,
+}
+
+type weekdays uint32
+
+// parseWeekdays creates a set of weekdays from a string slice
+func parseWeekdays(days []string) (weekdays, error) {
+	var result uint32
+	for _, day := range days {
+		if len(day) == 0 {
+			continue
+		}
+
+		weekday, err := parseWeekday(day)
+		if err != nil {
+			return weekdays(0), err
+		}
+		// #nosec G115 -- weekday is guaranteed to be between 0–6 by parseWeekday()
+		result |= 1 << uint32(weekday)
+	}
+
+	return weekdays(result), nil
+}
+
+// Contains returns true if the specified weekday is a member of this set.
+func (w weekdays) Contains(day time.Weekday) bool {
+	// #nosec G115 -- day is time.Weekday [0-6], shift safe within uint32
+	return uint32(w)&(1<<uint32(day)) != 0
+}
+
+// String returns a string representation of the set of weekdays.
+func (w weekdays) String() string {
+	var b strings.Builder
+	for i := uint32(0); i < 7; i++ {
+		if uint32(w)&(1<<i) != 0 {
+			b.WriteString(time.Weekday(i).String()[0:3])
+		} else {
+			b.WriteString("---")
+		}
+	}
+
+	return b.String()
+}
+
+func parseWeekday(day string) (time.Weekday, error) {
+	if n, err := strconv.Atoi(day); err == nil {
+		if n >= 0 && n < 7 {
+			return time.Weekday(n), nil
+		}
+		return time.Sunday, fmt.Errorf("invalid weekday, number out of range: %s", day)
+	}
+
+	if weekday, ok := dayStrings[strings.ToLower(day)]; ok {
+		return weekday, nil
+	}
+	return time.Sunday, fmt.Errorf("invalid weekday: %s", day)
+}

pkg/timewindow/days_test.go

@@ -0,0 +1,46 @@
+package timewindow
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseWeekdays(t *testing.T) {
+	tests := []struct {
+		input  string
+		result string
+	}{
+		{"0,4", "Sun---------Thu------"},
+		{"su,mo,tu", "SunMonTue------------"},
+		{"sunday,tu,thu", "Sun---Tue---Thu------"},
+		{"THURSDAY", "------------Thu------"},
+		{"we,WED,WeDnEsDaY", "---------Wed---------"},
+		{"", "---------------------"},
+		{",,,", "---------------------"},
+	}
+
+	for _, tst := range tests {
+		res, err := parseWeekdays(strings.Split(tst.input, ","))
+		if err != nil {
+			t.Errorf("Received error for input %s: %v", tst.input, err)
+		} else if res.String() != tst.result {
+			t.Errorf("Test %s: Expected %s got %s", tst.input, tst.result, res.String())
+		}
+	}
+}
+
+func TestParseWeekdaysErrors(t *testing.T) {
+	tests := []string{
+		"15",
+		"-8",
+		"8",
+		"mon,tue,wed,fridayyyy",
+	}
+
+	for _, tst := range tests {
+		_, err := parseWeekdays(strings.Split(tst, ","))
+		if err == nil {
+			t.Errorf("Expected to receive error for input %s", tst)
+		}
+	}
+}

pkg/timewindow/timewindow.go

@@ -0,0 +1,85 @@
+// Package timewindow provides utilities for handling days of the week,
+// including parsing, representing, and manipulating sets of weekdays.
+// It enables flexible specification of time windows for reboot operations
+// in kured, supporting various formats and convenience functions.
+package timewindow
+
+import (
+	"fmt"
+	"time"
+)
+
+// TimeWindow specifies a schedule of days and times.
+type TimeWindow struct {
+	days      weekdays
+	location  *time.Location
+	startTime time.Time
+	endTime   time.Time
+}
+
+// New creates a TimeWindow instance based on string inputs specifying a schedule.
+func New(days []string, startTime, endTime, location string) (*TimeWindow, error) {
+	tw := &TimeWindow{}
+
+	var err error
+	if tw.days, err = parseWeekdays(days); err != nil {
+		return nil, err
+	}
+
+	if tw.location, err = time.LoadLocation(location); err != nil {
+		return nil, err
+	}
+
+	if tw.startTime, err = parseTime(startTime, tw.location); err != nil {
+		return nil, err
+	}
+
+	if tw.endTime, err = parseTime(endTime, tw.location); err != nil {
+		return nil, err
+	}
+
+	return tw, nil
+}
+
+// Contains determines whether the specified time is within this time window.
+func (tw *TimeWindow) Contains(t time.Time) bool {
+	loctime := t.In(tw.location)
+	if !tw.days.Contains(loctime.Weekday()) {
+		return false
+	}
+
+	start := time.Date(loctime.Year(), loctime.Month(), loctime.Day(), tw.startTime.Hour(), tw.startTime.Minute(), tw.startTime.Second(), 0, tw.location)
+	end := time.Date(loctime.Year(), loctime.Month(), loctime.Day(), tw.endTime.Hour(), tw.endTime.Minute(), tw.endTime.Second(), 1e9-1, tw.location)
+
+	// Time Wrap validation
+	// First we check for start and end time, if start is after end time
+	// Next we need to validate if we want to wrap to the day before or to the day after
+	// For that we check the loctime value to see if it is before end time, we wrap with the day before
+	// Otherwise we wrap to the next day.
+	if tw.startTime.After(tw.endTime) {
+		if loctime.Before(end) {
+			start = start.Add(-24 * time.Hour)
+		} else {
+			end = end.Add(24 * time.Hour)
+		}
+	}
+
+	return (loctime.After(start) || loctime.Equal(start)) && (loctime.Before(end) || loctime.Equal(end))
+}
+
+// String returns a string representation of this time window.
+func (tw *TimeWindow) String() string {
+	return fmt.Sprintf("%s between %02d:%02d and %02d:%02d %s", tw.days.String(), tw.startTime.Hour(), tw.startTime.Minute(), tw.endTime.Hour(), tw.endTime.Minute(), tw.location.String())
+}
+
+// parseTime tries to parse a time with several formats.
+func parseTime(s string, loc *time.Location) (time.Time, error) {
+	fmts := []string{"15:04", "15:04:05", "03:04pm", "15", "03pm", "3pm"}
+	for _, f := range fmts {
+		if t, err := time.ParseInLocation(f, s, loc); err == nil {
+			return t, nil
+		}
+	}
+
+	return time.Now(), fmt.Errorf("invalid time format: %s", s)
+}

pkg/timewindow/timewindow_test.go

@@ -0,0 +1,97 @@
+package timewindow
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestTimeWindows(t *testing.T) {
+	type testcase struct {
+		time   string
+		result bool
+	}
+
+	tests := []struct {
+		days  string
+		start string
+		end   string
+		loc   string
+		cases []testcase
+	}{
+		{"mon,tue,wed,thu,fri", "9am", "5pm", "America/Los_Angeles", []testcase{
+			{"2019/03/31 10:00 PDT", false},
+			{"2019/04/04 00:49 PDT", false},
+			{"2019/04/04 12:00 PDT", true},
+			{"2019/04/04 11:59 UTC", false},
+			{"2019/04/05 08:59 PDT", false},
+			{"2019/04/05 9:01 PDT", true},
+		}},
+		{"mon,we,fri", "10:01", "11:30am", "America/Los_Angeles", []testcase{
+			{"2019/04/05 10:30 PDT", true},
+			{"2019/04/06 10:30 PDT", false},
+			{"2019/04/07 10:30 PDT", false},
+			{"2019/04/08 10:30 PDT", true},
+			{"2019/04/09 10:30 PDT", false},
+			{"2019/04/10 10:30 PDT", true},
+			{"2019/04/11 10:30 PDT", false},
+		}},
+		{"mo,tu,we,th,fr", "00:00", "23:59:59", "UTC", []testcase{
+			{"2019/04/18 00:00 UTC", true},
+			{"2019/04/18 23:59 UTC", true},
+		}},
+		{"mon,tue,wed,thu,fri", "9pm", "5am", "America/Los_Angeles", []testcase{
+			{"2019/03/30 04:00 PDT", false},
+			{"2019/03/31 10:00 PDT", false},
+			{"2019/03/31 22:00 PDT", false},
+			{"2019/04/04 00:49 PDT", true},
+			{"2019/04/04 12:00 PDT", false},
+			{"2019/04/04 22:49 PDT", true},
+			{"2019/04/05 00:49 PDT", true},
+			{"2019/04/05 08:59 PDT", false},
+			{"2019/04/05 9:01 PDT", false},
+		}},
+		{"mon,tue,wed,thu,fri", "11:59pm", "00:01am", "America/Los_Angeles", []testcase{
+			{"2019/04/04 23:58 PDT", false},
+			{"2019/04/04 23:59 PDT", true},
+			{"2019/04/05 00:00 PDT", true},
+			{"2019/04/05 00:01 PDT", true},
+			{"2019/04/05 00:02 PDT", false},
+		}},
+		{"mon,tue,wed,fri", "11:59pm", "00:01am", "America/Los_Angeles", []testcase{
+			{"2019/04/04 23:58 PDT", false},
+			{"2019/04/04 23:59 PDT", false}, // Even that this falls in the between the hours Thursday is not included so should not run
+			{"2019/04/05 00:00 PDT", true},
+			{"2019/04/05 00:02 PDT", false},
+		}},
+		{"mon,tue,wed,thu", "11:59pm", "00:01am", "America/Los_Angeles", []testcase{
+			{"2019/04/04 23:58 PDT", false},
+			{"2019/04/04 23:59 PDT", true},
+			{"2019/04/05 00:00 PDT", false}, // Even that this falls in the between the hours Friday is not included so should not run
+			{"2019/04/05 00:02 PDT", false},
+		}},
+		{"mon,tue,wed,thu,fri", "11:59pm", "00:01am", "UTC", []testcase{
+			{"2019/04/04 23:58 UTC", false},
+			{"2019/04/04 23:59 UTC", true},
+			{"2019/04/05 00:00 UTC", true},
+			{"2019/04/05 00:01 UTC", true},
+			{"2019/04/05 00:02 UTC", false},
+		}},
+	}
+
+	for i, tst := range tests {
+		tw, err := New(strings.Split(tst.days, ","), tst.start, tst.end, tst.loc)
+		if err != nil {
+			t.Errorf("Test [%d] failed to create TimeWindow: %v", i, err)
+		}
+
+		for _, cas := range tst.cases {
+			tm, err := time.ParseInLocation("2006/01/02 15:04 MST", cas.time, tw.location)
+			if err != nil {
+				t.Errorf("Failed to parse time \"%s\": %v", cas.time, err)
+			} else if cas.result != tw.Contains(tm) {
+				t.Errorf("(%s) contains (%s) didn't match expected result of %v", tw.String(), cas.time, cas.result)
+			}
+		}
+	}
+}

tests/kind/main_test.go

@@ -0,0 +1,429 @@
+package kind
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"os/exec"
+	"strconv"
+	"testing"
+	"time"
+)
+
+const (
+	kuredDevImage string = "kured:dev"
+)
+
+// KindTest cluster deployed by each TestMain function, prepared to run a given test scenario.
+type KindTest struct {
+	kindConfigPath  string
+	clusterName     string
+	timeout         time.Duration
+	deployManifests []string
+	localImages     []string
+	logsDir         string
+	logBuffer       bytes.Buffer
+	testInstance    *testing.T // Maybe move this to testing.TB
+}
+
+func (k *KindTest) Write(p []byte) (n int, err error) {
+	k.testInstance.Helper()
+	k.logBuffer.Write(p)
+	return len(p), nil
+}
+
+func (k *KindTest) FlushLog() {
+	k.testInstance.Helper()
+	k.testInstance.Log(k.logBuffer.String())
+	k.logBuffer.Reset()
+}
+
+func (k *KindTest) RunCmd(cmdDetails ...string) error {
+	cmd := exec.Command(cmdDetails[0], cmdDetails[1:]...)
+	// by making KindTest a Writer, we can simply wire k to logs
+	// writing to k will write to proper logs.
+	cmd.Stdout = k
+	cmd.Stderr = k
+
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Option that can be passed to the NewKind function in order to change the configuration
+// of the test cluster
+type Option func(k *KindTest)
+
+// Deploy can be passed to NewKind to deploy extra components, in addition to the base deployment.
+func Deploy(manifest string) Option {
+	return func(k *KindTest) {
+		k.deployManifests = append(k.deployManifests, manifest)
+	}
+}
+
+// ExportLogs can be passed to NewKind to specify the folder where the kubernetes logs will be exported after the tests.
+func ExportLogs(folder string) Option {
+	return func(k *KindTest) {
+		k.logsDir = folder
+	}
+}
+
+// Timeout for long-running operations (e.g. deployments, readiness probes...)
+func Timeout(t time.Duration) Option {
+	return func(k *KindTest) {
+		k.timeout = t
+	}
+}
+
+// LocalImage is passed to NewKind to allow loading a local Docker image into the cluster
+func LocalImage(nameTag string) Option {
+	return func(k *KindTest) {
+		k.localImages = append(k.localImages, nameTag)
+	}
+}
+
+// NewKind creates a kind cluster given a name and set of Option instances.
+func NewKindTester(kindClusterName string, filePath string, t *testing.T, options ...Option) *KindTest {
+
+	k := &KindTest{
+		clusterName:    kindClusterName,
+		timeout:        10 * time.Minute,
+		kindConfigPath: filePath,
+		testInstance:   t,
+	}
+	for _, option := range options {
+		option(k)
+	}
+	return k
+}
+
+// Prepare the kind cluster.
+func (k *KindTest) Create() error {
+	err := k.RunCmd("kind", "create", "cluster", "--name", k.clusterName, "--config", k.kindConfigPath)
+
+	if err != nil {
+		return fmt.Errorf("failed to create cluster: %v", err)
+	}
+
+	for _, img := range k.localImages {
+		if err := k.RunCmd("kind", "load", "docker-image", "--name", k.clusterName, img); err != nil {
+			return fmt.Errorf("failed to load image: %v", err)
+		}
+	}
+	for _, mf := range k.deployManifests {
+		kubectlContext := fmt.Sprintf("kind-%v", k.clusterName)
+		if err := k.RunCmd("kubectl", "--context", kubectlContext, "apply", "-f", mf); err != nil {
+			return fmt.Errorf("failed to deploy manifest: %v", err)
+		}
+	}
+	return nil
+}
+
+func (k *KindTest) Destroy() error {
+	if k.logsDir != "" {
+		if err := k.RunCmd("kind", "export", "logs", k.logsDir, "--name", k.clusterName); err != nil {
+			return fmt.Errorf("failed to export logs: %v. will not teardown", err)
+		}
+	}
+
+	if err := k.RunCmd("kind", "delete", "cluster", "--name", k.clusterName); err != nil {
+		return fmt.Errorf("failed to destroy cluster: %v", err)
+	}
+	return nil
+}
+
+func TestE2EWithCommand(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"previous",
+		"current",
+		"next",
+	}
+	// Iterate over each Kubernetes version
+	for _, version := range kindClusterConfigs {
+		version := version
+		// Define a subtest for each combination
+		t.Run(version, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-command-%v-%v", version, randomInt)
+			kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds.yaml"))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
+				t.Fatalf("failed to create sentinels: %v", err)
+			}
+
+			if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
+				t.Fatalf("failed to follow reboot: %v", err)
+			}
+		})
+	}
+}
+
+func TestE2EWithSignal(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"previous",
+		"current",
+		"next",
+	}
+	// Iterate over each Kubernetes version
+	for _, version := range kindClusterConfigs {
+		version := version
+		// Define a subtest for each combination
+		t.Run(version, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-signal-%v-%v", version, randomInt)
+			kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds-signal.yaml"))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
+				t.Fatalf("failed to create sentinels: %v", err)
+			}
+
+			if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
+				t.Fatalf("failed to follow reboot: %v", err)
+			}
+		})
+	}
+}
+
+func TestE2EConcurrentWithCommand(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"previous",
+		"current",
+		"next",
+	}
+	// Iterate over each Kubernetes version
+	for _, version := range kindClusterConfigs {
+		version := version
+		// Define a subtest for each combination
+		t.Run(version, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-concurrentcommand-%v-%v", version, randomInt)
+			kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds-concurrent-command.yaml"))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
+				t.Fatalf("failed to create sentinels: %v", err)
+			}
+
+			if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
+				t.Fatalf("failed to follow reboot: %v", err)
+			}
+		})
+	}
+}
+
+func TestE2EConcurrentWithSignal(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"previous",
+		"current",
+		"next",
+	}
+	// Iterate over each Kubernetes version
+	for _, version := range kindClusterConfigs {
+		version := version
+		// Define a subtest for each combination
+		t.Run(version, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-concurrentsignal-%v-%v", version, randomInt)
+			kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds-concurrent-signal.yaml"))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
+				t.Fatalf("failed to create sentinels: %v", err)
+			}
+
+			if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
+				t.Fatalf("failed to follow reboot: %v", err)
+			}
+		})
+	}
+}
+
+func TestCordonningIsKept(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"concurrency1",
+		"concurrency2",
+	}
+	// Iterate over each test variant
+	for _, variant := range kindClusterConfigs {
+		variant := variant
+		// Define a subtest for each combination
+		t.Run(variant, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-cordon-%v-%v", variant, randomInt)
+			kindClusterConfigFile := "../../.github/kind-cluster-next.yaml"
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			var manifest string
+			if variant == "concurrency1" {
+				manifest = "testfiles/kured-ds-signal.yaml"
+			} else {
+				manifest = "testfiles/kured-ds-concurrent-signal.yaml"
+			}
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy(manifest))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", "testfiles/node-stays-as-cordonned.sh", kindContext); err != nil {
+				t.Fatalf("node did not reboot in time: %v", err)
+			}
+		})
+	}
+}
+func TestE2EBlocker(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"podblocker",
+	}
+	// Iterate over each variant of the test
+	for _, variant := range kindClusterConfigs {
+		variant := variant
+		// Define a subtest for each combination
+		t.Run(variant, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-cordon-%v-%v", variant, randomInt)
+			kindClusterConfigFile := "../../.github/kind-cluster-next.yaml"
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy(fmt.Sprintf("testfiles/kured-ds-%v.yaml", variant)))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", fmt.Sprintf("testfiles/%v.sh", variant), kindContext); err != nil {
+				t.Fatalf("node blocker test did not succeed: %v", err)
+			}
+		})
+	}
+}

tests/kind/test-metrics.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+expected="$1"
+if [[ "$expected" != "0" && "$expected" != "1" ]]; then
+    echo "You should give an argument to this script, the gauge value (0 or 1)"
+    exit 1
+fi
+
+HOST="${HOST:-localhost}"
+PORT="${PORT:-30000}"
+NODENAME="${NODENAME-chart-testing-control-plane}"
+
+reboot_required=$(docker exec "$NODENAME" curl "http://$HOST:$PORT/metrics" | awk '/^kured_reboot_required/{print $2}')
+if [[ "$reboot_required" == "$expected" ]]; then
+    echo "Test success"
+else
+    echo "Test failed"
+    exit 1
+fi

tests/kind/testfiles/create-reboot-sentinels.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+kubectl_flags=( )
+[[ "$1" != "" ]] && kubectl_flags=("${kubectl_flags[@]}" --context "$1")
+
+# To speed up the system, let's not kill the control plane.
+for nodename in $(${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes -o name | grep -v control-plane); do
+    echo "Creating reboot sentinel on $nodename"
+    docker exec "${nodename/node\//}" hostname
+    docker exec "${nodename/node\//}" touch "${SENTINEL_FILE:-/var/run/reboot-required}"
+done

tests/kind/testfiles/follow-coordinated-reboot.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+REBOOTCOUNT=${REBOOTCOUNT:-2} # By default we only create two sentinels in create-reboot-sentinels.
+DEBUG="${DEBUG:-false}"
+CONTAINER_NAME_FORMAT=${CONTAINER_NAME_FORMAT:-"chart-testing-*"}
+
+kubectl_flags=( )
+[[ "$1" != "" ]] && kubectl_flags=("${kubectl_flags[@]}" --context "$1")
+
+tmp_dir=$(mktemp -d -t kured-XXXX)
+
+function gather_logs_and_cleanup {
+    if [[ -f "$tmp_dir"/node_output ]]; then
+        rm "$tmp_dir"/node_output
+    fi
+    rmdir "$tmp_dir"
+
+    # The next commands are useful regardless of success or failures.
+    if [[ "$DEBUG" == "true" ]]; then
+        echo "############################################################"
+        # This is useful to see if containers have crashed.
+        echo "docker ps -a:"
+        docker ps -a
+	      echo "docker journal logs"
+	      journalctl -u docker --no-pager
+
+        # This is useful to see if the nodes have _properly_ rebooted.
+        # It should show the reboot/two container starts per node.
+        for id in $(docker ps -a -q); do
+            echo "############################################################"
+            echo "docker logs for container $id:"
+            docker logs "$id"
+        done
+
+    fi
+}
+trap gather_logs_and_cleanup EXIT
+
+declare -A was_unschedulable
+declare -A has_recovered
+max_attempts="200"
+sleep_time=5
+attempt_num=1
+
+# Get docker info of each of those kind containers. If one has crashed, restart it.
+
+set +o errexit
+echo "There are $REBOOTCOUNT nodes total needing reboot in the cluster"
+until [ ${#was_unschedulable[@]} == "$REBOOTCOUNT" ] && [ ${#has_recovered[@]} == "$REBOOTCOUNT" ]
+do
+    echo "${#was_unschedulable[@]} nodes were removed from pool once:" "${!was_unschedulable[@]}"
+    echo "${#has_recovered[@]} nodes removed from the pool are now back:" "${!has_recovered[@]}"
+
+
+    ${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes -o custom-columns=NAME:.metadata.name,SCHEDULABLE:.spec.unschedulable --no-headers | grep -v control-plane > "$tmp_dir"/node_output
+    if [[ "$DEBUG" == "true" ]]; then
+        # This is useful to see if a node gets stuck after drain, and doesn't
+        # come back up.
+        echo "Result of command kubectl unschedulable nodes:"
+        cat "$tmp_dir"/node_output
+    fi
+
+    while read -r node; do
+        unschedulable=$(echo "$node" | grep true | cut -f 1 -d ' ')
+        if [ -n "$unschedulable" ] && [ -z ${was_unschedulable["$unschedulable"]+x} ] ; then
+            echo "$unschedulable is now unschedulable!"
+            was_unschedulable["$unschedulable"]=1
+        fi
+        schedulable=$(echo "$node" | grep '<none>' | cut -f 1 -d ' ')
+        if [ -n "$schedulable" ] && [ ${was_unschedulable["$schedulable"]+x} ] && [ -z ${has_recovered["$schedulable"]+x} ]; then
+            echo "$schedulable has recovered!"
+            has_recovered["$schedulable"]=1
+        fi
+
+        # If the container has crashed, restart it.
+        node_name=$(echo "$node" | cut -f 1 -d ' ')
+        stopped_container_id=$(docker container ls --filter=name="$node_name" --filter=status=exited -q)
+        if [ -n "$stopped_container_id" ]; then echo "Node $stopped_container_id needs restart"; docker start "$stopped_container_id"; echo "Container started."; fi
+
+    done < "$tmp_dir"/node_output
+
+    if [[ "${#has_recovered[@]}" == "$REBOOTCOUNT" ]]; then
+        echo "All nodes recovered."
+        break
+    else
+        if (( attempt_num == max_attempts ))
+        then
+            echo "Attempt $attempt_num failed and there are no more attempts left!"
+            exit 1
+        else
+            echo "Attempt $attempt_num failed! Trying again in $sleep_time seconds..."
+            sleep "$sleep_time"
+        fi
+    fi
+    (( attempt_num++ ))
+done
+
+echo "Test successful"

tests/kind/testfiles/node-stays-as-cordonned.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+kubectl_flags=( )
+[[ "$1" != "" ]] && kubectl_flags=("${kubectl_flags[@]}" --context "$1")
+
+cordon() {
+  kubectl "${kubectl_flags[@]}" cordon "${precordonned_node}"
+}
+
+create_sentinel() {
+  docker exec "${precordonned_node}" touch "${SENTINEL_FILE:-/var/run/reboot-required}"
+  docker exec "${notcordonned_node}" touch "${SENTINEL_FILE:-/var/run/reboot-required}"
+}
+
+check_reboot_required() {
+  while true;
+  do
+    docker exec "${precordonned_node}" stat /var/run/reboot-required > /dev/null && echo "Reboot still required" || return 0
+    sleep 3
+  done
+}
+
+check_node_back_online_as_cordonned() {
+  sleep 5 # For safety, wait for 5 seconds, so that the kubectl command succeeds.
+  # This test might be giving us false positive until we work on reliability of the
+  # test.
+  while true;
+  do
+    result=$(kubectl "${kubectl_flags[@]}" get node "${precordonned_node}" --no-headers | awk '{print $2;}')
+    test "${result}" != "Ready,SchedulingDisabled" && echo "Node ${precordonned_node} in state ${result}" || return 0
+    sleep 3
+  done
+}
+
+check_node_back_online_as_uncordonned() {
+  while true;
+  do
+    result=$(kubectl "${kubectl_flags[@]}" get node "${notcordonned_node}" --no-headers | awk '{print $2;}')
+    test "${result}" != "Ready" && echo "Node ${notcordonned_node} in state ${result}" || return 0
+    sleep 3
+  done
+}
+### Start main
+
+worker_nodes=$(${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes -o custom-columns=name:metadata.name --no-headers | grep worker)
+precordonned_node=$(echo "$worker_nodes" | head -n 1)
+notcordonned_node=$(echo "$worker_nodes" | tail -n 1)
+
+# Wait for kured to install correctly
+sleep 15
+cordon
+create_sentinel
+check_reboot_required
+echo "Node has rebooted, but may take time to come back ready"
+check_node_back_online_as_cordonned
+check_node_back_online_as_uncordonned
+echo "Showing final node state"
+${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes
+echo "Test successful"

tests/kind/testfiles/podblocker.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+kubectl_flags=( )
+[[ "$1" != "" ]] && kubectl_flags=("${kubectl_flags[@]}" --context "$1")
+
+function gather_logs_and_cleanup {
+      for id in $(docker ps -q); do
+          echo "############################################################"
+          echo "docker logs for container $id:"
+          docker logs "$id"
+      done
+      ${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" logs ds/kured --all-pods -n kube-system
+}
+trap gather_logs_and_cleanup EXIT
+
+set +o errexit
+worker=$(${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes -o custom-columns=name:metadata.name --no-headers | grep worker | head -n 1)
+
+${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" label nodes "$worker" blocked-host=yes
+
+${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" apply -f - << EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+  labels:
+    app: blocker
+spec:
+  containers:
+    - name: nginx
+      image: nginx
+      imagePullPolicy: IfNotPresent
+  nodeSelector:
+    blocked-host: "yes"
+EOF
+
+docker exec "$worker" touch "${SENTINEL_FILE:-/var/run/reboot-required}"
+
+set -o errexit
+max_attempts="100"
+attempt_num=1
+sleep_time=5
+
+until ${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" logs ds/kured --all-pods -n kube-system  | grep -i -e "Reboot.*blocked"
+do
+  if (( attempt_num == max_attempts )); then
+      echo "Attempt $attempt_num failed and there are no more attempts left!"
+      exit 1
+  else
+      echo "Did not find 'reboot blocked' in the log, retrying in $sleep_time seconds (Attempt #$attempt_num)"
+      sleep "$sleep_time"
+  fi
+  (( attempt_num++ ))
+done