Merge pull request #1085 from dholbach/prep-1.17.1

prepare 1.17.1 release

.github/kind-cluster-1.28.yaml

@@ -1,13 +0,0 @@
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  image: "kindest/node:v1.28.9"
-- role: control-plane
-  image: "kindest/node:v1.28.9"
-- role: control-plane
-  image: "kindest/node:v1.28.9"
-- role: worker
-  image: "kindest/node:v1.28.9"
-- role: worker
-  image: "kindest/node:v1.28.9"

.github/kind-cluster-1.29.yaml

@@ -1,13 +0,0 @@
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  image: "kindest/node:v1.29.4"
-- role: control-plane
-  image: "kindest/node:v1.29.4"
-- role: control-plane
-  image: "kindest/node:v1.29.4"
-- role: worker
-  image: "kindest/node:v1.29.4"
-- role: worker
-  image: "kindest/node:v1.29.4"

.github/kind-cluster-1.30.yaml

@@ -1,13 +0,0 @@
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  image: "kindest/node:v1.30.2"
-- role: control-plane
-  image: "kindest/node:v1.30.2"
-- role: control-plane
-  image: "kindest/node:v1.30.2"
-- role: worker
-  image: "kindest/node:v1.30.2"
-- role: worker
-  image: "kindest/node:v1.30.2"

.github/kind-cluster-current.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: "kindest/node:v1.30.10"
+- role: worker
+  image: "kindest/node:v1.30.10"
+- role: worker
+  image: "kindest/node:v1.30.10"

.github/kind-cluster-next.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: "kindest/node:v1.31.6"
+- role: worker
+  image: "kindest/node:v1.31.6"
+- role: worker
+  image: "kindest/node:v1.31.6"

.github/kind-cluster-previous.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: "kindest/node:v1.29.14"
+- role: worker
+  image: "kindest/node:v1.29.14"
+- role: worker
+  image: "kindest/node:v1.29.14"

.github/workflows/codeql.yml

@@ -21,6 +21,9 @@ on:
   schedule:
     - cron: '24 13 * * 3'
 
+permissions:
+  contents: read
+
 jobs:
   analyze:
     name: Analyze
@@ -38,12 +41,17 @@ jobs:
         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
 
     steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -57,7 +65,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -70,6 +78,6 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/dependency-review.yml

@@ -0,0 +1,27 @@
+# Dependency Review Action
+#
+# This Action will scan dependency manifest files that change as part of a Pull Request,
+# surfacing known-vulnerable versions of the packages declared or updated in the PR.
+# Once installed, if the workflow run is marked as required,
+# PRs introducing known-vulnerable packages will be blocked from merging.
+#
+# Source repository: https://github.com/actions/dependency-review-action
+name: 'Dependency Review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - name: 'Checkout Repository'
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0

.github/workflows/on-main-push.yaml

@@ -10,6 +10,9 @@ env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
+permissions:
+  contents: read
+
 jobs:
   tag-scan-and-push-final-image:
     name: "Build, scan, and publish tagged image"
@@ -19,16 +22,21 @@ jobs:
       contents: write
       packages: write
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden Runner
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Ensure go version
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           go-version-file: 'go.mod'
           check-latest: true
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -36,15 +44,15 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
 
       - name: Find current tag version
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
@@ -59,7 +67,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build image
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           context: .
           platforms: linux/arm64, linux/amd64, linux/arm/v7, linux/arm/v6, linux/386
@@ -70,13 +78,11 @@ jobs:
 
       - name: Generate SBOM
         run: |
-          .tmp/syft ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }} -o spdx > kured.sbom
+          hack/bin/syft ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }} -o spdx > kured.sbom
 
       - name: Sign and attest artifacts
         run: |
-          .tmp/cosign sign -y -r ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
-
-          .tmp/cosign sign-blob -y --output-signature kured.sbom.sig --output-certificate kured.sbom.pem kured.sbom
-
-          .tmp/cosign attest -y --type spdx --predicate kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
-          .tmp/cosign attach sbom --type spdx --sbom kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
+          hack/bin/cosign sign -y -r ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
+          hack/bin/cosign sign-blob -y --output-signature kured.sbom.sig --output-certificate kured.sbom.pem kured.sbom
+          hack/bin/cosign attest -y --type spdx --predicate kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}
+          hack/bin/cosign attach sbom --type spdx --sbom kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.sha_short }}

.github/workflows/on-pr.yaml

@@ -4,59 +4,43 @@ on:
   push:
 
 jobs:
-  pr-gotest:
-    name: Run go tests
+  pr-short-tests:
+    name: Run short go tests
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Ensure go version
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           go-version-file: 'go.mod'
           check-latest: true
       - name: run tests
-        run: go test -json ./... > test.json
+        run: make test
       - name: Annotate tests
         if: always()
-        uses: guyarb/golang-test-annoations@v0.8.0
+        uses: guyarb/golang-test-annoations@2941118d7ef622b1b3771d1ff6eae9e90659eb26 # v0.8.0
         with:
           test-results: test.json
 
-  pr-shellcheck:
-    name: Lint bash code with shellcheck
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: Run ShellCheck
-      uses: bewuethr/shellcheck-action@v2
-
-  pr-lint-code:
-    name: Lint golang code
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: Ensure go version
-      uses: actions/setup-go@v5
-      with:
-        go-version-file: 'go.mod'
-        check-latest: true
-    - name: Lint cmd folder
-      uses: Jerome1337/golint-action@v1.0.3
-      with:
-        golint-path: './cmd/...'
-    - name: Lint pkg folder
-      uses: Jerome1337/golint-action@v1.0.3
-      with:
-        golint-path: './pkg/...'
 
   pr-check-docs-links:
     name: Check docs for incorrect links
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden Runner
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Link Checker
-      uses: lycheeverse/lychee-action@2b973e86fc7b1f6b36a93795fe2c9c6ae1118621
+      uses: lycheeverse/lychee-action@f613c4a64e50d792e0b31ec34bbcbba12263c6a6
       env:
         GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
       with:
@@ -70,25 +54,30 @@ jobs:
     name: Build image and scan it against known vulnerabilities
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden Runner
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Ensure go version
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           go-version-file: 'go.mod'
           check-latest: true
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
       - name: Setup GoReleaser
         run: make bootstrap-tools
       - name: Find current tag version
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
         id: tags
       - name: Build image
-        run: VERSION="${{ steps.tags.outputs.sha_short }}" make image
+        run: VERSION="${{ steps.tags.outputs.sha_short }}" DH_ORG="${{ github.repository_owner }}" make image
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0
         with:
           image-ref: 'ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }}'
           format: 'table'
@@ -104,251 +93,90 @@ jobs:
   #     (compared to helm charts, manifests cannot easily template changes based on versions)
   # Helm charts are _trailing_ releases, while manifests are done during development.
   # This test uses the "command" reboot-method.
-  e2e-manifests-command:
-    name: End-to-End test with kured with code and manifests from HEAD (command)
+  e2e-manifests:
+    name: End-to-End test with kured with code and manifests from HEAD
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        kubernetes:
-          - "1.28"
-          - "1.29"
-          - "1.30"
+        testname:
+          - "TestE2EWithCommand"
+          - "TestE2EWithSignal"
+          - "TestE2EConcurrentWithCommand"
+          - "TestE2EConcurrentWithSignal"
+        kubernetes_version:
+          - "previous"
+          - "current"
+          - "next"
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden Runner
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Ensure go version
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           go-version-file: 'go.mod'
           check-latest: true
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
       - name: Setup GoReleaser
         run: make bootstrap-tools
       - name: Find current tag version
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
         id: tags
-      - name: Build artifacts
-        run: |
-          VERSION="${{ steps.tags.outputs.sha_short }}" make image
-          VERSION="${{ steps.tags.outputs.sha_short }}" make manifest
-
-      - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions
-        run: |
-          sudo bash << EOF
-              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
-              echo '{}' > /etc/docker/daemon.json
-              systemctl restart docker || journalctl --no-pager -n 500
-              systemctl status docker
-          EOF
-
-      # Default name for helm/kind-action kind clusters is "chart-testing"
-      - name: Create kind cluster with 5 nodes
-        uses: helm/kind-action@v1.10.0
+      - name: Install kind
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
         with:
-          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
-          version: v0.14.0
-
-      - name: Preload previously built images onto kind cluster
-        run: kind load docker-image ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }} --name chart-testing
-
-      - name: Do not wait for an hour before detecting the rebootSentinel
-        run: |
-          sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds.yaml
-
-      - name: Install kured with kubectl
-        run: |
-          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml
-
-      - name: Ensure kured is ready
-        uses: nick-invision/retry@v3.0.0
-        with:
-          timeout_minutes: 10
-          max_attempts: 10
-          retry_wait_seconds: 60
-          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
-          command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
-
-      - name: Create reboot sentinel files
-        run: |
-          ./tests/kind/create-reboot-sentinels.sh
-
-      - name: Follow reboot until success
-        env:
-          DEBUG: true
-        run: |
-          ./tests/kind/follow-coordinated-reboot.sh
+          install_only: true
+          version: v0.27.0
+      - name: Run specific e2e tests
+        run: make e2e-test ARGS="-run ^${{ matrix.testname }}/${{ matrix.kubernetes_version }}"
 
 
-  # This ensures the latest code works with the manifests built from tree.
-  # It is useful for two things:
-  # - Test manifests changes (obviously), ensuring they don't break existing clusters
-  # - Ensure manifests work with the latest versions even with no manifest change
-  #     (compared to helm charts, manifests cannot easily template changes based on versions)
-  # Helm charts are _trailing_ releases, while manifests are done during development.
-  # This test uses the "signal" reboot-method.
-  e2e-manifests-signal:
-    name: End-to-End test with kured with code and manifests from HEAD (signal)
+  e2e-tests-singleversion:
+    name: End-to-End test targetting a single version of kubernetes
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        kubernetes:
-          - "1.28"
-          - "1.29"
-          - "1.30"
+        testname:
+          - "TestCordonningIsKept/concurrency1"
+          - "TestCordonningIsKept/concurrency2"
+          - "TestE2EBlocker/podblocker"
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden Runner
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Ensure go version
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           go-version-file: 'go.mod'
           check-latest: true
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
       - name: Setup GoReleaser
         run: make bootstrap-tools
       - name: Find current tag version
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
         id: tags
-      - name: Build artifacts
-        run: |
-          VERSION="${{ steps.tags.outputs.sha_short }}" make image
-          VERSION="${{ steps.tags.outputs.sha_short }}" make manifest
-
-      - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions
-        run: |
-          sudo bash << EOF
-              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
-              echo '{}' > /etc/docker/daemon.json
-              systemctl restart docker || journalctl --no-pager -n 500
-              systemctl status docker
-          EOF
-
-      # Default name for helm/kind-action kind clusters is "chart-testing"
-      - name: Create kind cluster with 5 nodes
-        uses: helm/kind-action@v1.10.0
+      - name: Install kind
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
         with:
-          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
-          version: v0.14.0
-
-      - name: Preload previously built images onto kind cluster
-        run: kind load docker-image ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }} --name chart-testing
-
-      - name: Do not wait for an hour before detecting the rebootSentinel
-        run: |
-          sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds-signal.yaml
-
-      - name: Install kured with kubectl
-        run: |
-          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds-signal.yaml
-
-      - name: Ensure kured is ready
-        uses: nick-invision/retry@v3.0.0
-        with:
-          timeout_minutes: 10
-          max_attempts: 10
-          retry_wait_seconds: 60
-          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
-          command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
-
-      - name: Create reboot sentinel files
-        run: |
-          ./tests/kind/create-reboot-sentinels.sh
-
-      - name: Follow reboot until success
-        env:
-          DEBUG: true
-        run: |
-          ./tests/kind/follow-coordinated-reboot.sh
-
-
-
-  # This ensures the latest code works with the manifests built from tree.
-  # It is useful for two things:
-  # - Test manifests changes (obviously), ensuring they don't break existing clusters
-  # - Ensure manifests work with the latest versions even with no manifest change
-  #     (compared to helm charts, manifests cannot easily template changes based on versions)
-  # Helm charts are _trailing_ releases, while manifests are done during development.
-  # Concurrency = 2
-  e2e-manifests-concurent:
-    name: End-to-End test with kured with code and manifests from HEAD (concurrent)
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        kubernetes:
-          - "1.28"
-          - "1.29"
-          - "1.30"
-    steps:
-      - uses: actions/checkout@v4
-      - name: Ensure go version
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: 'go.mod'
-          check-latest: true
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Setup GoReleaser
-        run: make bootstrap-tools
-      - name: Find current tag version
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-        id: tags
-      - name: Build artifacts
-        run: |
-          VERSION="${{ steps.tags.outputs.sha_short }}" make image
-          VERSION="${{ steps.tags.outputs.sha_short }}" make manifest
-
-      - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions
-        run: |
-          sudo bash << EOF
-              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
-              echo '{}' > /etc/docker/daemon.json
-              systemctl restart docker || journalctl --no-pager -n 500
-              systemctl status docker
-          EOF
-
-      # Default name for helm/kind-action kind clusters is "chart-testing"
-      - name: Create kind cluster with 5 nodes
-        uses: helm/kind-action@v1.10.0
-        with:
-          config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml
-          version: v0.14.0
-
-      - name: Preload previously built images onto kind cluster
-        run: kind load docker-image ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }} --name chart-testing
-
-      - name: Do not wait for an hour before detecting the rebootSentinel
-        run: |
-          sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds.yaml
-          sed -i 's/#\(.*\)--concurrency=1/\1--concurrency=2/g' kured-ds.yaml
-
-      - name: Install kured with kubectl
-        run: |
-          kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml
-
-      - name: Ensure kured is ready
-        uses: nick-invision/retry@v3.0.0
-        with:
-          timeout_minutes: 10
-          max_attempts: 10
-          retry_wait_seconds: 60
-          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
-          command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
-
-      - name: Create reboot sentinel files
-        run: |
-          ./tests/kind/create-reboot-sentinels.sh
-
-      - name: Follow reboot until success
-        env:
-          DEBUG: true
-        run: |
-          ./tests/kind/follow-coordinated-reboot.sh
+          install_only: true
+          version: v0.27.0
+          # Keep this until v1.31 (or superior) becomes the default kubectl version for the kind-action.
+          # It is used in podblocker shell script test to use --all-pods.
+          # If the podblocker e2e test relies on another way, this can also be removed.
+          kubectl_version: v1.31.0
+      - name: Run specific e2e tests
+        run: make e2e-test ARGS="-run ^${{ matrix.testname }}"

.github/workflows/on-tag.yaml

@@ -12,6 +12,9 @@ env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
+permissions:
+  contents: read
+
 jobs:
   tag-scan-and-push-final-image:
     name: "Build, scan, and publish tagged image"
@@ -21,9 +24,14 @@ jobs:
       contents: write
       packages: write
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden Runner
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Ensure go version
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           go-version-file: 'go.mod'
           check-latest: true
@@ -31,9 +39,9 @@ jobs:
         run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
         id: tags
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
       - name: Setup GoReleaser
         run: make bootstrap-tools
       - name: Build binaries
@@ -41,7 +49,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Build single image for scan
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           context: .
           platforms: linux/amd64
@@ -51,7 +59,7 @@ jobs:
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0
         with:
           image-ref: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}'
           format: 'table'
@@ -61,7 +69,7 @@ jobs:
           severity: 'CRITICAL,HIGH'
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -69,12 +77,12 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Build release images
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           context: .
           platforms: linux/arm64, linux/amd64, linux/arm/v7, linux/arm/v6, linux/386
@@ -85,13 +93,11 @@ jobs:
 
       - name: Generate SBOM
         run: |
-          .tmp/syft ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }} -o spdx > kured.sbom
+          hack/bin/syft ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }} -o spdx > kured.sbom
 
       - name: Sign and attest artifacts
         run: |
-          .tmp/cosign sign -y -r ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
-
-          .tmp/cosign sign-blob -y --output-signature kured.sbom.sig kured.sbom
-
-          .tmp/cosign attest -y --type spdx --predicate kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
-          .tmp/cosign attach sbom --type spdx --sbom kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
+          hack/bin/cosign sign -y -r ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
+          hack/bin/cosign sign-blob -y --output-signature kured.sbom.sig kured.sbom
+          hack/bin/cosign attest -y --type spdx --predicate kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}
+          hack/bin/cosign attach sbom --type spdx --sbom kured.sbom ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tags.outputs.version }}

.github/workflows/periodics-daily.yaml

@@ -9,13 +9,18 @@ jobs:
     name: Run go tests
     runs-on: ubuntu-latest
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: run tests
-        run: go test -json ./... > test.json
+        run: make test
       - name: Annotate tests
         if: always()
-        uses: guyarb/golang-test-annoations@v0.8.0
+        uses: guyarb/golang-test-annoations@2941118d7ef622b1b3771d1ff6eae9e90659eb26 # v0.8.0
         with:
           test-results: test.json
 
@@ -25,7 +30,12 @@ jobs:
     steps:
     # Stale by default waits for 60 days before marking PR/issues as stale, and closes them after 21 days.
     # Do not expire the first issues that would allow the community to grow.
-    - uses: actions/stale@v9
+    - name: Harden Runner
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue was automatically considered stale due to lack of activity. Please update it and/or join our slack channels to promote it, before it automatically closes (in 7 days).'
@@ -39,9 +49,14 @@ jobs:
     name: Check docs for incorrect links
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden Runner
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Link Checker
-      uses: lycheeverse/lychee-action@2b973e86fc7b1f6b36a93795fe2c9c6ae1118621
+      uses: lycheeverse/lychee-action@f613c4a64e50d792e0b31ec34bbcbba12263c6a6
       env:
         GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
       with:
@@ -52,25 +67,30 @@ jobs:
     name: Build image and scan it against known vulnerabilities
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden Runner
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Ensure go version
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           go-version-file: 'go.mod'
           check-latest: true
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
       - name: Setup GoReleaser
         run: make bootstrap-tools
       - name: Find current tag version
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
         id: tags
       - name: Build artifacts
-        run: VERSION="${{ steps.tags.outputs.sha_short }}" make image
+        run: VERSION="${{ steps.tags.outputs.sha_short }}" DH_ORG="${{ github.repository_owner }}" make image
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0
         with:
           image-ref: 'ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.sha_short }}'
           format: 'table'

.github/workflows/scorecard.yml

@@ -0,0 +1,78 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '34 3 * * 6'
+  push:
+    branches: [ "main" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - name: "Checkout code"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v3.pre.node20
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard (optional).
+      # Commenting out will disable upload of results to your repo's Code Scanning dashboard
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
+        with:
+          sarif_file: results.sarif

.gitignore

@@ -2,4 +2,6 @@ cmd/kured/kured
 vendor
 build
 dist
-.tmp
+test.json
+tests/kind/testfiles/*.yaml
+hack/bin/

CONTRIBUTING.md

@@ -5,13 +5,13 @@ Slack][slack], reporting or triaging [issues][issues] or contributing code
 to `kured`.
 
 In any case, it will make sense to familiarise yourself with the main
-[README][readme] to understand the different features and options, which is
-helpful for testing. The "building" section in particular makes sense if
-you are planning to contribute code.
+[documentation][documentation] to understand the different features and
+options, which is helpful for testing. The "building" section in
+particular makes sense if you are planning to contribute code.
 
-[slack]: README.md#getting-help
+[slack]: https://github.com/kubereboot/kured/blob/main/README.md#getting-help
 [issues]: https://github.com/kubereboot/kured/issues
-[readme]: README.md
+[documentation]: https://kured.dev/docs
 
 ## Certificate of Origin
 
@@ -41,6 +41,15 @@ All Kured repositories are kept under <https://github.com/kubereboot>. To find t
 | <https://github.com/kubereboot/charts>  | Helm chart                |
 | <https://github.com/kubereboot/website> | website and documentation |
 
+### Kured code
+
+- Kured's main code can be found in the [`cmd`](cmd) and [`pkg`](pkg) directories
+- Its e2e tests are in the [`tests`](tests) directory
+- We use [GoReleaser to build](.goreleaser.yml).
+- Every PR and tagged release is tested by [Kind in GitHub workflows](.github/workflows).
+
+As a project, we try to follow all the official and obvious standards.
+
 ## Regular development activities
 
 ### Prepare environment
@@ -66,16 +75,23 @@ efbb0c3: Document version compatibility in release notes
 
 Search the git log for inspiration for your cases.
 
-Please update our `.github/workflows` with the new k8s images, starting by
-the creation of a `.github/kind-cluster-<version>.yaml`, then updating
-our workflows with the new versions.
+Please update our `.github/workflows` with the new k8s images.
 
-Once you updated everything, make sure you update the support matrix on
-the main [README][readme] as well.
+For that, run the following:
+
+`cp .github/kind-cluster-current.yaml .github/kind-cluster-previous.yaml`
+`cp .github/kind-cluster-next.yaml .github/kind-cluster-current.yaml`
+
+Then edit `.github/kind-cluster-next.yaml` to point to the new version.
+
+This will make the full test matrix updated (the CI and the test code).
+
+Once your code passes all tests, update the support matrix in
+the [installation docs](https://kured.dev/docs/installation/).
 
 ### Updating other dependencies
 
-Dependabot proposes changes in our go.mod/go.sum.
+Dependabot proposes changes in our `go.mod`/`go.sum`.
 Some of those changes are covered by CI testing, some are not.
 
 Please make sure to test those not covered by CI (mostly the integration
@@ -87,7 +103,7 @@ We run periodic jobs (see also Automated testing section of this documentation).
 Those should be monitored for failures.
 
 If a failure happen in periodics, something terribly wrong must have happened
-(or github is failing at the creation of a kind cluster). Please monitor those
+(or GitHub is failing at the creation of a kind cluster). Please monitor those
 failures carefully.
 
 ### Introducing new features
@@ -107,23 +123,24 @@ This also means that when you expose a new feature, you should create another PR
 for your changes in <https://github.com/kubereboot/charts> to make your feature
 available at the next kured version for helm users.
 
-In the charts PR, you can directly bump the appVersion to the next minor version
+In the charts PR, you can directly bump the `appVersion` to the next minor version
 (you are introducing a new feature, which requires a bump of the minor number.
-For example, if current appVersion is 1.6.x, make sure you update your appVersion
-to 1.7.0). It allows us to have an easy view of what we land each release.
+For example, if current `appVersion` is `1.6.x`, make sure you update your `appVersion`
+to `1.7.0`). It allows us to have an easy view of what we land each release.
 
 Do not hesitate to increase the test coverage for your feature, whether it's unit
-testing to full functional testing (even using helm charts)
+testing to full functional testing (even using helm charts).
 
 ### Increasing test coverage
 
 We are welcoming any change to increase our test coverage.
-See also our github issues for the label `testing`.
+See also our GitHub issues for the label
+[`testing`](https://github.com/kubereboot/kured/labels/testing).
 
 ## Automated testing
 
-Our CI is covered by github actions.
-You can see their contents in .github/workflows.
+Our CI is covered by GitHub actions.
+You can see their contents in `.github/workflows`.
 
 We currently run:
 
@@ -137,6 +154,13 @@ To test your code manually, follow the section Manual testing.
 
 ## Manual (release) testing
 
+### Quick Golang code testing
+
+Please run `make test` to run only the basic tests. It gives a good
+idea of the code behaviour.
+
+### Manual functional testing
+
 Before `kured` is released, we want to make sure it still works fine on the
 previous, current and next minor version of Kubernetes (with respect to the
 `client-go` & `kubectl` dependencies in use). For local testing e.g.
@@ -152,15 +176,11 @@ results, if you login to a node and run:
 sudo touch /var/run/reboot-required
 ```
 
-### Example of golang testing
-
-Please run `make test`. You should have `golint` installed.
-
-### Example of testing with `minikube`
+### Example of functional testing with `minikube`
 
 A test-run with `minikube` could look like this:
 
-```console
+```cli
 # start minikube
 minikube start --driver=kvm2 --kubernetes-version <k8s-release>
 
@@ -192,7 +212,7 @@ Then you can check for the lock release.
 
 A test-run with `kind` could look like this:
 
-```console
+```cli
 # create kind cluster
 kind create cluster --config .github/kind-cluster-<k8s-version>.yaml
 
@@ -204,22 +224,44 @@ kind create cluster --config .github/kind-cluster-<k8s-version>.yaml
 
 ```
 
+### Example of testing with `kind` and `make`
+
+A test-run with `kind` and `make` can be done with the following command:
+
+```cli
+# Build kured:dev image, build manifests, and run the "long" go tests
+make e2e-test
+```
+
+You can alter test behaviour by passing arguments to this command.
+A few examples below:
+
+```shell
+# Run only TestE2EWithSignal test for the kubernetes version named "current" (see kind file)
+make e2e-test ARGS="-run ^TestE2EWithSignal/current"
+# Run all tests but make sure to extend the timeout, for slower machines.
+make e2e-test ARGS="-timeout 1200s'
+```
+
 ## Publishing a new kured release
 
 ### Prepare Documentation
 
-Check that [compatibility matrix](https://kured.dev/docs/installation/) is updated
-to the new version you want to release.
+Ensure the [compatibility matrix](https://kured.dev/docs/installation/) is
+updated to the new version you want to release.
 
-### Create a tag on the repo
+### Update the manifests with the new version
 
-Before going further, we should freeze the code for a release, by
-tagging the code. The Github-Action should start a new job and push
-the new image to the registry.
+Create a commit updating the manifest with future image [like this one](https://github.com/kubereboot/kured/commit/58091f6145771f426b4b9e012a43a9c847af2560).
 
-### Create the combined manifest
+### Create the new version tag on the repo
 
-Now create the `kured-<release>-dockerhub.yaml` for e.g. `1.3.0`:
+Tag the previously created commit with the future release version.
+The Github Actions workflow will push the new image to the registry.
+
+### Create the combined manifest for the new version
+
+Now create the `kured-<new version>-dockerhub.yaml` for e.g. `1.3.0`:
 
 ```sh
 VERSION=1.3.0
@@ -229,13 +271,23 @@ cat kured-rbac.yaml > "$MANIFEST"
 cat kured-ds.yaml >> "$MANIFEST"
 ```
 
-### Publish release artifacts
+### Publish new version release artifacts
 
-Now you can head to the Github UI, use the version number as tag and upload the
-`kured-<release>-dockerhub.yaml` file.
+Now you can head to the GitHub UI for releases, drafting a new
+release. Chose, as tag, the new version number.
+
+Click to generate the release notes.
+
+Fill, as name, "Kured <new version>".
+
+Edit the generated text.
 
 Please describe what's new and noteworthy in the release notes, list the PRs
 that landed and give a shout-out to everyone who contributed.
-
 Please also note down on which releases the upcoming `kured` release was
-tested on. (Check old release notes if you're unsure.)
+tested on or what it supports. (Check old release notes if you're unsure.)
+
+Before clicking on publishing release, upload the yaml manifest
+(`kured-<new version>-dockerhub.yaml`) file.
+
+Click on publish the release and set as the latest release.

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=$TARGETPLATFORM alpine:3.20.2 as bin
+FROM alpine:3.21.3@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c AS bin
 
 ARG TARGETOS
 ARG TARGETARCH
@@ -19,7 +19,7 @@ RUN set -ex \
     esac \
   && cp /dist/kured_${TARGETOS}_${TARGETARCH}${SUFFIX}/kured /dist/kured;
 
-FROM --platform=$TARGETPLATFORM alpine:3.20.2
+FROM alpine:3.21.3@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c
 RUN apk update --no-cache && apk upgrade --no-cache && apk add --no-cache ca-certificates tzdata
 COPY --from=bin /dist/kured /usr/bin/kured
 ENTRYPOINT ["/usr/bin/kured"]

MAINTAINERS

@@ -1,5 +1,5 @@
 Christian Hopf <christian.kotzbauer@gmail.com> (@ckotzbauer)
 Daniel Holbach <daniel.holbach@gmail.com> (@dholbach)
-Hidde Beydals <hidde@weave.works> (@hiddeco)
+Hidde Beydals <hidde@hhh.computer> (@hiddeco)
 Jack Francis <jackfrancis@gmail.com> (@jackfrancis)
 Jean-Philippe Evrard <open-source@a.spamming.party> (@evrardjp)

Makefile

@@ -1,41 +1,62 @@
 .DEFAULT: all
 .PHONY: all clean image minikube-publish manifest test kured-all
 
-TEMPDIR=./.tmp
-GORELEASER_CMD=$(TEMPDIR)/goreleaser
-DH_ORG=kubereboot
+HACKDIR=./hack/bin
+GORELEASER_CMD=$(HACKDIR)/goreleaser
+DH_ORG ?= kubereboot
 VERSION=$(shell git rev-parse --short HEAD)
 SUDO=$(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 
 all: image
 
-$(TEMPDIR):
-	mkdir -p $(TEMPDIR)
+$(HACKDIR):
+	mkdir -p $(HACKDIR)
 
 .PHONY: bootstrap-tools
-bootstrap-tools: $(TEMPDIR)
-	VERSION=v1.24.0 TMPDIR=.tmp bash .github/scripts/goreleaser-install.sh
-	curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b .tmp v1.0.1
-	curl -sSfL https://github.com/sigstore/cosign/releases/download/v2.2.3/cosign-linux-amd64 -o .tmp/cosign
-	chmod +x .tmp/goreleaser .tmp/cosign .tmp/syft
+bootstrap-tools: $(HACKDIR)
+	command -v $(HACKDIR)/goreleaser || VERSION=v1.24.0 TMPDIR=$(HACKDIR) bash hack/installers/goreleaser-install.sh
+	command -v  $(HACKDIR)/syft || curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b $(HACKDIR) v1.0.1
+	command -v  $(HACKDIR)/cosign || curl -sSfL https://github.com/sigstore/cosign/releases/download/v2.2.3/cosign-linux-amd64 -o $(HACKDIR)/cosign
+	command -v  $(HACKDIR)/shellcheck || (curl -sSfL https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz | tar -J -v -x shellcheck-stable/shellcheck && mv shellcheck-stable/shellcheck $(HACKDIR)/shellcheck && rmdir shellcheck-stable)
+	chmod +x $(HACKDIR)/goreleaser $(HACKDIR)/cosign $(HACKDIR)/syft $(HACKDIR)/shellcheck
+	command -v staticcheck || go install honnef.co/go/tools/cmd/staticcheck@latest
 
 clean:
 	rm -rf ./dist
 
-kured:
+kured: bootstrap-tools
 	$(GORELEASER_CMD) build --clean --single-target --snapshot
 
-kured-all:
+kured-all: bootstrap-tools
 	$(GORELEASER_CMD) build --clean --snapshot
 
-kured-release-tag:
+kured-release-tag: bootstrap-tools
 	$(GORELEASER_CMD) release --clean
 
-kured-release-snapshot:
+kured-release-snapshot: bootstrap-tools
 	$(GORELEASER_CMD) release --clean --snapshot
 
 image: kured
-	$(SUDO) docker buildx build --load -t ghcr.io/$(DH_ORG)/kured:$(VERSION) .
+	$(SUDO) docker buildx build --no-cache --load -t ghcr.io/$(DH_ORG)/kured:$(VERSION) .
+
+dev-image: image
+	$(SUDO) docker tag ghcr.io/$(DH_ORG)/kured:$(VERSION) kured:dev
+
+dev-manifest:
+	# basic e2e scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' kured-ds.yaml > tests/kind/testfiles/kured-ds.yaml
+	# signal e2e scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' kured-ds-signal.yaml > tests/kind/testfiles/kured-ds-signal.yaml
+	# concurrency e2e command scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' -e 's/#\(.*\)--concurrency=1/\1--concurrency=2/g' kured-ds.yaml > tests/kind/testfiles/kured-ds-concurrent-command.yaml
+	# concurrency e2e signal scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' -e 's/#\(.*\)--concurrency=1/\1--concurrency=2/g' kured-ds-signal.yaml > tests/kind/testfiles/kured-ds-concurrent-signal.yaml
+	# pod blocker e2e signal scenario
+	sed -e "s#image: ghcr.io/.*kured.*#image: kured:dev#g" -e 's/#\(.*\)--period=1h/\1--period=20s/g' -e 's/#\(.*\)--blocking-pod-selector=name=temperamental/\1--blocking-pod-selector=app=blocker/g' kured-ds-signal.yaml > tests/kind/testfiles/kured-ds-podblocker.yaml
+
+e2e-test: dev-manifest dev-image
+	echo "Running ALL go tests"
+	go test -count=1 -v --parallel 4 ./... $(ARGS)
 
 minikube-publish: image
 	$(SUDO) docker save ghcr.io/$(DH_ORG)/kured | (eval $$(minikube docker-env) && docker load)
@@ -45,10 +66,9 @@ manifest:
 	sed -i "s#image: ghcr.io/.*kured.*#image: ghcr.io/$(DH_ORG)/kured:$(VERSION)#g" kured-ds-signal.yaml
 	echo "Please generate combined manifest if necessary"
 
-test:
-	echo "Running go tests"
-	go test ./...
-	echo "Running golint on pkg"
-	golint ./pkg/...
-	echo "Running golint on cmd"
-	golint ./cmd/...
+test: bootstrap-tools
+	echo "Running short go tests"
+	go test -test.short -json ./... > test.json
+	echo "Running shellcheck"
+	find . -name '*.sh' | xargs -n1 $(HACKDIR)/shellcheck
+	staticcheck ./...

cmd/kured/main.go

@@ -8,37 +8,32 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"os/exec"
 	"reflect"
-	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/containrrr/shoutrrr"
+	"github.com/kubereboot/kured/internal"
+	"github.com/kubereboot/kured/pkg/blockers"
+	"github.com/kubereboot/kured/pkg/checkers"
+	"github.com/kubereboot/kured/pkg/daemonsetlock"
+	"github.com/kubereboot/kured/pkg/delaytick"
+	"github.com/kubereboot/kured/pkg/reboot"
+	"github.com/kubereboot/kured/pkg/taints"
+	"github.com/kubereboot/kured/pkg/timewindow"
 	papi "github.com/prometheus/client_golang/api"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-	"github.com/spf13/viper"
+	flag "github.com/spf13/pflag"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	kubectldrain "k8s.io/kubectl/pkg/drain"
-
-	"github.com/google/shlex"
-
-	shoutrrr "github.com/containrrr/shoutrrr"
-	"github.com/kubereboot/kured/pkg/alerts"
-	"github.com/kubereboot/kured/pkg/daemonsetlock"
-	"github.com/kubereboot/kured/pkg/delaytick"
-	"github.com/kubereboot/kured/pkg/reboot"
-	"github.com/kubereboot/kured/pkg/taints"
-	"github.com/kubereboot/kured/pkg/timewindow"
-	"github.com/kubereboot/kured/pkg/util"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
@@ -63,7 +58,7 @@ var (
 	lockReleaseDelay                time.Duration
 	prometheusURL                   string
 	preferNoScheduleTaintName       string
-	alertFilter                     *regexp.Regexp
+	alertFilter                     regexpValue
 	alertFilterMatchOnly            bool
 	alertFiringOnly                 bool
 	rebootSentinelFile              string
@@ -108,11 +103,6 @@ const (
 	// EnvPrefix The environment variable prefix of all environment variables bound to our command line flags.
 	EnvPrefix = "KURED"
 
-	// MethodCommand is used as "--reboot-method" value when rebooting with the configured "--reboot-command"
-	MethodCommand = "command"
-	// MethodSignal is used as "--reboot-method" value when rebooting with a SIGRTMIN+5 signal.
-	MethodSignal = "signal"
-
 	sigTrminPlus5 = 34 + 5
 )
 
@@ -121,138 +111,185 @@ func init() {
 }
 
 func main() {
-	cmd := NewRootCommand()
 
-	if err := cmd.Execute(); err != nil {
-		log.Fatal(err)
-	}
-}
-
-// NewRootCommand construct the Cobra root command
-func NewRootCommand() *cobra.Command {
-	rootCmd := &cobra.Command{
-		Use:               "kured",
-		Short:             "Kubernetes Reboot Daemon",
-		PersistentPreRunE: bindViper,
-		PreRun:            flagCheck,
-		Run:               root}
-
-	rootCmd.PersistentFlags().StringVar(&nodeID, "node-id", "",
+	flag.StringVar(&nodeID, "node-id", "",
 		"node name kured runs on, should be passed down from spec.nodeName via KURED_NODE_ID environment variable")
-	rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false,
+	flag.BoolVar(&forceReboot, "force-reboot", false,
 		"force a reboot even if the drain fails or times out")
-	rootCmd.PersistentFlags().StringVar(&metricsHost, "metrics-host", "",
+	flag.StringVar(&metricsHost, "metrics-host", "",
 		"host where metrics will listen")
-	rootCmd.PersistentFlags().IntVar(&metricsPort, "metrics-port", 8080,
+	flag.IntVar(&metricsPort, "metrics-port", 8080,
 		"port number where metrics will listen")
-	rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1,
+	flag.IntVar(&drainGracePeriod, "drain-grace-period", -1,
 		"time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used")
-	rootCmd.PersistentFlags().StringVar(&drainPodSelector, "drain-pod-selector", "",
+	flag.StringVar(&drainPodSelector, "drain-pod-selector", "",
 		"only drain pods with labels matching the selector (default: '', all pods)")
-	rootCmd.PersistentFlags().IntVar(&skipWaitForDeleteTimeoutSeconds, "skip-wait-for-delete-timeout", 0,
+	flag.IntVar(&skipWaitForDeleteTimeoutSeconds, "skip-wait-for-delete-timeout", 0,
 		"when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node")
-	rootCmd.PersistentFlags().DurationVar(&drainDelay, "drain-delay", 0,
+	flag.DurationVar(&drainDelay, "drain-delay", 0,
 		"delay drain for this duration (default: 0, disabled)")
-	rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 0,
+	flag.DurationVar(&drainTimeout, "drain-timeout", 0,
 		"timeout after which the drain is aborted (default: 0, infinite time)")
-	rootCmd.PersistentFlags().DurationVar(&rebootDelay, "reboot-delay", 0,
+	flag.DurationVar(&rebootDelay, "reboot-delay", 0,
 		"delay reboot for this duration (default: 0, disabled)")
-	rootCmd.PersistentFlags().StringVar(&rebootMethod, "reboot-method", "command",
+	flag.StringVar(&rebootMethod, "reboot-method", "command",
 		"method to use for reboots. Available: command")
-	rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60,
+	flag.DurationVar(&period, "period", time.Minute*60,
 		"sentinel check period")
-	rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system",
+	flag.StringVar(&dsNamespace, "ds-namespace", "kube-system",
 		"namespace containing daemonset on which to place lock")
-	rootCmd.PersistentFlags().StringVar(&dsName, "ds-name", "kured",
+	flag.StringVar(&dsName, "ds-name", "kured",
 		"name of daemonset on which to place lock")
-	rootCmd.PersistentFlags().StringVar(&lockAnnotation, "lock-annotation", KuredNodeLockAnnotation,
+	flag.StringVar(&lockAnnotation, "lock-annotation", KuredNodeLockAnnotation,
 		"annotation in which to record locking node")
-	rootCmd.PersistentFlags().DurationVar(&lockTTL, "lock-ttl", 0,
+	flag.DurationVar(&lockTTL, "lock-ttl", 0,
 		"expire lock annotation after this duration (default: 0, disabled)")
-	rootCmd.PersistentFlags().DurationVar(&lockReleaseDelay, "lock-release-delay", 0,
+	flag.DurationVar(&lockReleaseDelay, "lock-release-delay", 0,
 		"delay lock release for this duration (default: 0, disabled)")
-	rootCmd.PersistentFlags().StringVar(&prometheusURL, "prometheus-url", "",
+	flag.StringVar(&prometheusURL, "prometheus-url", "",
 		"Prometheus instance to probe for active alerts")
-	rootCmd.PersistentFlags().Var(&regexpValue{&alertFilter}, "alert-filter-regexp",
+	flag.Var(&alertFilter, "alert-filter-regexp",
 		"alert names to ignore when checking for active alerts")
-	rootCmd.PersistentFlags().BoolVar(&alertFilterMatchOnly, "alert-filter-match-only", false,
+	flag.BoolVar(&alertFilterMatchOnly, "alert-filter-match-only", false,
 		"Only block if the alert-filter-regexp matches active alerts")
-	rootCmd.PersistentFlags().BoolVar(&alertFiringOnly, "alert-firing-only", false,
+	flag.BoolVar(&alertFiringOnly, "alert-firing-only", false,
 		"only consider firing alerts when checking for active alerts")
-	rootCmd.PersistentFlags().StringVar(&rebootSentinelFile, "reboot-sentinel", "/var/run/reboot-required",
+	flag.StringVar(&rebootSentinelFile, "reboot-sentinel", "/var/run/reboot-required",
 		"path to file whose existence triggers the reboot command")
-	rootCmd.PersistentFlags().StringVar(&preferNoScheduleTaintName, "prefer-no-schedule-taint", "",
+	flag.StringVar(&preferNoScheduleTaintName, "prefer-no-schedule-taint", "",
 		"Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Disabled by default. Set e.g. to \"weave.works/kured-node-reboot\" to enable tainting.")
-	rootCmd.PersistentFlags().StringVar(&rebootSentinelCommand, "reboot-sentinel-command", "",
+	flag.StringVar(&rebootSentinelCommand, "reboot-sentinel-command", "",
 		"command for which a zero return code will trigger a reboot command")
-	rootCmd.PersistentFlags().StringVar(&rebootCommand, "reboot-command", "/bin/systemctl reboot",
+	flag.StringVar(&rebootCommand, "reboot-command", "/bin/systemctl reboot",
 		"command to run when a reboot is required")
-	rootCmd.PersistentFlags().IntVar(&concurrency, "concurrency", 1,
+	flag.IntVar(&concurrency, "concurrency", 1,
 		"amount of nodes to concurrently reboot. Defaults to 1")
-	rootCmd.PersistentFlags().IntVar(&rebootSignal, "reboot-signal", sigTrminPlus5,
+	flag.IntVar(&rebootSignal, "reboot-signal", sigTrminPlus5,
 		"signal to use for reboot, SIGRTMIN+5 by default.")
-
-	rootCmd.PersistentFlags().StringVar(&slackHookURL, "slack-hook-url", "",
+	flag.StringVar(&slackHookURL, "slack-hook-url", "",
 		"slack hook URL for reboot notifications [deprecated in favor of --notify-url]")
-	rootCmd.PersistentFlags().StringVar(&slackUsername, "slack-username", "kured",
+	flag.StringVar(&slackUsername, "slack-username", "kured",
 		"slack username for reboot notifications")
-	rootCmd.PersistentFlags().StringVar(&slackChannel, "slack-channel", "",
+	flag.StringVar(&slackChannel, "slack-channel", "",
 		"slack channel for reboot notifications")
-	rootCmd.PersistentFlags().StringVar(&notifyURL, "notify-url", "",
+	flag.StringVar(&notifyURL, "notify-url", "",
 		"notify URL for reboot notifications (cannot use with --slack-hook-url flags)")
-	rootCmd.PersistentFlags().StringVar(&messageTemplateUncordon, "message-template-uncordon", "Node %s rebooted & uncordoned successfully!",
+	flag.StringVar(&messageTemplateUncordon, "message-template-uncordon", "Node %s rebooted & uncordoned successfully!",
 		"message template used to notify about a node being successfully uncordoned")
-	rootCmd.PersistentFlags().StringVar(&messageTemplateDrain, "message-template-drain", "Draining node %s",
+	flag.StringVar(&messageTemplateDrain, "message-template-drain", "Draining node %s",
 		"message template used to notify about a node being drained")
-	rootCmd.PersistentFlags().StringVar(&messageTemplateReboot, "message-template-reboot", "Rebooting node %s",
+	flag.StringVar(&messageTemplateReboot, "message-template-reboot", "Rebooting node %s",
 		"message template used to notify about a node being rebooted")
-
-	rootCmd.PersistentFlags().StringArrayVar(&podSelectors, "blocking-pod-selector", nil,
+	flag.StringArrayVar(&podSelectors, "blocking-pod-selector", nil,
 		"label selector identifying pods whose presence should prevent reboots")
-
-	rootCmd.PersistentFlags().StringSliceVar(&rebootDays, "reboot-days", timewindow.EveryDay,
+	flag.StringSliceVar(&rebootDays, "reboot-days", timewindow.EveryDay,
 		"schedule reboot on these days")
-	rootCmd.PersistentFlags().StringVar(&rebootStart, "start-time", "0:00",
+	flag.StringVar(&rebootStart, "start-time", "0:00",
 		"schedule reboot only after this time of day")
-	rootCmd.PersistentFlags().StringVar(&rebootEnd, "end-time", "23:59:59",
+	flag.StringVar(&rebootEnd, "end-time", "23:59:59",
 		"schedule reboot only before this time of day")
-	rootCmd.PersistentFlags().StringVar(&timezone, "time-zone", "UTC",
+	flag.StringVar(&timezone, "time-zone", "UTC",
 		"use this timezone for schedule inputs")
-
-	rootCmd.PersistentFlags().BoolVar(&annotateNodes, "annotate-nodes", false,
+	flag.BoolVar(&annotateNodes, "annotate-nodes", false,
 		"if set, the annotations 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' will be given to nodes undergoing kured reboots")
-
-	rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "text",
+	flag.StringVar(&logFormat, "log-format", "text",
 		"use text or json log format")
-
-	rootCmd.PersistentFlags().StringSliceVar(&preRebootNodeLabels, "pre-reboot-node-labels", nil,
+	flag.StringSliceVar(&preRebootNodeLabels, "pre-reboot-node-labels", nil,
 		"labels to add to nodes before cordoning")
-	rootCmd.PersistentFlags().StringSliceVar(&postRebootNodeLabels, "post-reboot-node-labels", nil,
+	flag.StringSliceVar(&postRebootNodeLabels, "post-reboot-node-labels", nil,
 		"labels to add to nodes after uncordoning")
 
-	return rootCmd
+	flag.Parse()
+
+	// Load flags from environment variables
+	LoadFromEnv()
+
+	log.Infof("Kubernetes Reboot Daemon: %s", version)
+
+	if logFormat == "json" {
+		log.SetFormatter(&log.JSONFormatter{})
+	}
+
+	if nodeID == "" {
+		log.Fatal("KURED_NODE_ID environment variable required")
+	}
+	log.Infof("Node ID: %s", nodeID)
+
+	notifyURL = validateNotificationURL(notifyURL, slackHookURL)
+
+	err := validateNodeLabels(preRebootNodeLabels, postRebootNodeLabels)
+	if err != nil {
+		log.Warnf(err.Error())
+	}
+
+	log.Infof("PreferNoSchedule taint: %s", preferNoScheduleTaintName)
+
+	// This should be printed from blocker list instead of only blocking pod selectors
+	log.Infof("Blocking Pod Selectors: %v", podSelectors)
+
+	log.Infof("Reboot period %v", period)
+	log.Infof("Concurrency: %v", concurrency)
+
+	if annotateNodes {
+		log.Infof("Will annotate nodes during kured reboot operations")
+	}
+
+	// Now call the rest of the main loop.
+	window, err := timewindow.New(rebootDays, rebootStart, rebootEnd, timezone)
+	if err != nil {
+		log.Fatalf("Failed to build time window: %v", err)
+	}
+	log.Infof("Reboot schedule: %v", window)
+
+	log.Infof("Reboot method: %s", rebootMethod)
+	rebooter, err := internal.NewRebooter(rebootMethod, rebootCommand, rebootSignal)
+	if err != nil {
+		log.Fatalf("Failed to build rebooter: %v", err)
+	}
+
+	rebootChecker, err := internal.NewRebootChecker(rebootSentinelCommand, rebootSentinelFile)
+	if err != nil {
+		log.Fatalf("Failed to build reboot checker: %v", err)
+	}
+
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var blockCheckers []blockers.RebootBlocker
+	if prometheusURL != "" {
+		blockCheckers = append(blockCheckers, blockers.NewPrometheusBlockingChecker(papi.Config{Address: prometheusURL}, alertFilter.Regexp, alertFiringOnly, alertFilterMatchOnly))
+	}
+	if podSelectors != nil {
+		blockCheckers = append(blockCheckers, blockers.NewKubernetesBlockingChecker(client, nodeID, podSelectors))
+	}
+	log.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation)
+	if lockTTL > 0 {
+		log.Infof("Lock TTL set, lock will expire after: %v", lockTTL)
+	} else {
+		log.Info("Lock TTL not set, lock will remain until being released")
+	}
+	if lockReleaseDelay > 0 {
+		log.Infof("Lock release delay set, lock release will be delayed by: %v", lockReleaseDelay)
+	} else {
+		log.Info("Lock release delay not set, lock will be released immediately after rebooting")
+	}
+	lock := daemonsetlock.New(client, nodeID, dsNamespace, dsName, lockAnnotation, lockTTL, concurrency, lockReleaseDelay)
+
+	go rebootAsRequired(nodeID, rebooter, rebootChecker, blockCheckers, window, lock, client)
+	go maintainRebootRequiredMetric(nodeID, rebootChecker)
+
+	http.Handle("/metrics", promhttp.Handler())
+	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", metricsHost, metricsPort), nil))
 }
 
-// func that checks for deprecated slack-notification-related flags and node labels that do not match
-func flagCheck(cmd *cobra.Command, args []string) {
-	if slackHookURL != "" && notifyURL != "" {
-		log.Warnf("Cannot use both --notify-url and --slack-hook-url flags. Kured will use --notify-url flag only...")
-	}
-	if notifyURL != "" {
-		notifyURL = stripQuotes(notifyURL)
-	} else if slackHookURL != "" {
-		slackHookURL = stripQuotes(slackHookURL)
-		log.Warnf("Deprecated flag(s). Please use --notify-url flag instead.")
-		trataURL, err := url.Parse(slackHookURL)
-		if err != nil {
-			log.Warnf("slack-hook-url is not properly formatted... no notification will be sent: %v\n", err)
-		}
-		if len(strings.Split(strings.Trim(trataURL.Path, "/services/"), "/")) != 3 {
-			log.Warnf("slack-hook-url is not properly formatted... no notification will be sent: unexpected number of / in URL\n")
-		} else {
-			notifyURL = fmt.Sprintf("slack://%s", strings.Trim(trataURL.Path, "/services/"))
-		}
-	}
+func validateNodeLabels(preRebootNodeLabels []string, postRebootNodeLabels []string) error {
 	var preRebootNodeLabelKeys, postRebootNodeLabelKeys []string
 	for _, label := range preRebootNodeLabels {
 		preRebootNodeLabelKeys = append(preRebootNodeLabelKeys, strings.Split(label, "=")[0])
@@ -263,8 +300,95 @@ func flagCheck(cmd *cobra.Command, args []string) {
 	sort.Strings(preRebootNodeLabelKeys)
 	sort.Strings(postRebootNodeLabelKeys)
 	if !reflect.DeepEqual(preRebootNodeLabelKeys, postRebootNodeLabelKeys) {
-		log.Warnf("pre-reboot-node-labels keys and post-reboot-node-labels keys do not match. This may result in unexpected behaviour.")
+		return fmt.Errorf("pre-reboot-node-labels keys and post-reboot-node-labels keys do not match, resulting in unexpected behaviour")
 	}
+
+	return nil
+}
+
+func validateNotificationURL(notifyURL string, slackHookURL string) string {
+	switch {
+	case slackHookURL != "" && notifyURL != "":
+		log.Warnf("Cannot use both --notify-url (given: %v) and --slack-hook-url (given: %v) flags. Kured will only use --notify-url flag", slackHookURL, notifyURL)
+		return validateNotificationURL(notifyURL, "")
+	case notifyURL != "":
+		return stripQuotes(notifyURL)
+	case slackHookURL != "":
+		log.Warnf("Deprecated flag(s). Please use --notify-url flag instead.")
+		parsedURL, err := url.Parse(stripQuotes(slackHookURL))
+		if err != nil {
+			log.Errorf("slack-hook-url is not properly formatted... no notification will be sent: %v\n", err)
+			return ""
+		}
+		if len(strings.Split(strings.Replace(parsedURL.Path, "/services/", "", -1), "/")) != 3 {
+			log.Errorf("slack-hook-url is not properly formatted... no notification will be sent: unexpected number of / in URL\n")
+			return ""
+		}
+		return fmt.Sprintf("slack://%s", strings.Replace(parsedURL.Path, "/services/", "", -1))
+	}
+	return ""
+}
+
+// LoadFromEnv attempts to load environment variables corresponding to flags.
+// It looks for an environment variable with the uppercase version of the flag name (prefixed by EnvPrefix).
+func LoadFromEnv() {
+	flag.VisitAll(func(f *flag.Flag) {
+		envVarName := fmt.Sprintf("%s_%s", EnvPrefix, strings.ToUpper(strings.ReplaceAll(f.Name, "-", "_")))
+
+		if envValue, exists := os.LookupEnv(envVarName); exists {
+			switch f.Value.Type() {
+			case "int":
+				if parsedVal, err := strconv.Atoi(envValue); err == nil {
+					err := flag.Set(f.Name, strconv.Itoa(parsedVal))
+					if err != nil {
+						fmt.Printf("cannot set flag %s from env var named %s", f.Name, envVarName)
+						os.Exit(1)
+					} // Set int flag
+				} else {
+					fmt.Printf("Invalid value for env var named %s", envVarName)
+					os.Exit(1)
+				}
+			case "string":
+				err := flag.Set(f.Name, envValue)
+				if err != nil {
+					fmt.Printf("cannot set flag %s from env{%s}: %s\n", f.Name, envVarName, envValue)
+					os.Exit(1)
+				} // Set string flag
+			case "bool":
+				if parsedVal, err := strconv.ParseBool(envValue); err == nil {
+					err := flag.Set(f.Name, strconv.FormatBool(parsedVal))
+					if err != nil {
+						fmt.Printf("cannot set flag %s from env{%s}: %s\n", f.Name, envVarName, envValue)
+						os.Exit(1)
+					} // Set boolean flag
+				} else {
+					fmt.Printf("Invalid value for %s: %s\n", envVarName, envValue)
+					os.Exit(1)
+				}
+			case "duration":
+				// Set duration from the environment variable (e.g., "1h30m")
+				if _, err := time.ParseDuration(envValue); err == nil {
+					flag.Set(f.Name, envValue)
+				} else {
+					fmt.Printf("Invalid duration for %s: %s\n", envVarName, envValue)
+					os.Exit(1)
+				}
+			case "regexp":
+				// For regexp, set it from the environment variable
+				flag.Set(f.Name, envValue)
+			case "stringSlice":
+				// For stringSlice, split the environment variable by commas and set it
+				err := flag.Set(f.Name, envValue)
+				if err != nil {
+					fmt.Printf("cannot set flag %s from env{%s}: %s\n", f.Name, envVarName, envValue)
+					os.Exit(1)
+				}
+			default:
+				fmt.Printf("Unsupported flag type for %s\n", f.Name)
+			}
+		}
+	})
+
 }
 
 // stripQuotes removes any literal single or double quote chars that surround a string
@@ -280,218 +404,6 @@ func stripQuotes(str string) string {
 	return str
 }
 
-// bindViper initializes viper and binds command flags with environment variables
-func bindViper(cmd *cobra.Command, args []string) error {
-	v := viper.New()
-
-	v.SetEnvPrefix(EnvPrefix)
-	v.AutomaticEnv()
-	bindFlags(cmd, v)
-
-	return nil
-}
-
-// bindFlags binds each cobra flag to its associated viper configuration (environment variable)
-func bindFlags(cmd *cobra.Command, v *viper.Viper) {
-	cmd.Flags().VisitAll(func(f *pflag.Flag) {
-		// Environment variables can't have dashes in them, so bind them to their equivalent keys with underscores
-		if strings.Contains(f.Name, "-") {
-			v.BindEnv(f.Name, flagToEnvVar(f.Name))
-		}
-
-		// Apply the viper config value to the flag when the flag is not set and viper has a value
-		if !f.Changed && v.IsSet(f.Name) {
-			val := v.Get(f.Name)
-			log.Infof("Binding %s command flag to environment variable: %s", f.Name, flagToEnvVar(f.Name))
-			cmd.Flags().Set(f.Name, fmt.Sprintf("%v", val))
-		}
-	})
-}
-
-// flagToEnvVar converts command flag name to equivalent environment variable name
-func flagToEnvVar(flag string) string {
-	envVarSuffix := strings.ToUpper(strings.ReplaceAll(flag, "-", "_"))
-	return fmt.Sprintf("%s_%s", EnvPrefix, envVarSuffix)
-}
-
-// buildHostCommand writes a new command to run in the host namespace
-// Rancher based need different pid
-func buildHostCommand(pid int, command []string) []string {
-
-	// From the container, we nsenter into the proper PID to run the hostCommand.
-	// For this, kured daemonset need to be configured with hostPID:true and privileged:true
-	cmd := []string{"/usr/bin/nsenter", fmt.Sprintf("-m/proc/%d/ns/mnt", pid), "--"}
-	cmd = append(cmd, command...)
-	return cmd
-}
-
-func rebootRequired(sentinelCommand []string) bool {
-	cmd := util.NewCommand(sentinelCommand[0], sentinelCommand[1:]...)
-	if err := cmd.Run(); err != nil {
-		switch err := err.(type) {
-		case *exec.ExitError:
-			// We assume a non-zero exit code means 'reboot not required', but of course
-			// the user could have misconfigured the sentinel command or something else
-			// went wrong during its execution. In that case, not entering a reboot loop
-			// is the right thing to do, and we are logging stdout/stderr of the command
-			// so it should be obvious what is wrong.
-			if cmd.ProcessState.ExitCode() != 1 {
-				log.Warnf("sentinel command ended with unexpected exit code: %v", cmd.ProcessState.ExitCode())
-			}
-			return false
-		default:
-			// Something was grossly misconfigured, such as the command path being wrong.
-			log.Fatalf("Error invoking sentinel command: %v", err)
-		}
-	}
-	return true
-}
-
-// RebootBlocker interface should be implemented by types
-// to know if their instantiations should block a reboot
-type RebootBlocker interface {
-	isBlocked() bool
-}
-
-// PrometheusBlockingChecker contains info for connecting
-// to prometheus, and can give info about whether a reboot should be blocked
-type PrometheusBlockingChecker struct {
-	// prometheusClient to make prometheus-go-client and api config available
-	// into the PrometheusBlockingChecker struct
-	promClient *alerts.PromClient
-	// regexp used to get alerts
-	filter *regexp.Regexp
-	// bool to indicate if only firing alerts should be considered
-	firingOnly bool
-	// bool to indicate that we're only blocking on alerts which match the filter
-	filterMatchOnly bool
-}
-
-// KubernetesBlockingChecker contains info for connecting
-// to k8s, and can give info about whether a reboot should be blocked
-type KubernetesBlockingChecker struct {
-	// client used to contact kubernetes API
-	client   *kubernetes.Clientset
-	nodename string
-	// lised used to filter pods (podSelector)
-	filter []string
-}
-
-func (pb PrometheusBlockingChecker) isBlocked() bool {
-	alertNames, err := pb.promClient.ActiveAlerts(pb.filter, pb.firingOnly, pb.filterMatchOnly)
-	if err != nil {
-		log.Warnf("Reboot blocked: prometheus query error: %v", err)
-		return true
-	}
-	count := len(alertNames)
-	if count > 10 {
-		alertNames = append(alertNames[:10], "...")
-	}
-	if count > 0 {
-		log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
-		return true
-	}
-	return false
-}
-
-func (kb KubernetesBlockingChecker) isBlocked() bool {
-	fieldSelector := fmt.Sprintf("spec.nodeName=%s,status.phase!=Succeeded,status.phase!=Failed,status.phase!=Unknown", kb.nodename)
-	for _, labelSelector := range kb.filter {
-		podList, err := kb.client.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
-			LabelSelector: labelSelector,
-			FieldSelector: fieldSelector,
-			Limit:         10})
-		if err != nil {
-			log.Warnf("Reboot blocked: pod query error: %v", err)
-			return true
-		}
-
-		if len(podList.Items) > 0 {
-			podNames := make([]string, 0, len(podList.Items))
-			for _, pod := range podList.Items {
-				podNames = append(podNames, pod.Name)
-			}
-			if len(podList.Continue) > 0 {
-				podNames = append(podNames, "...")
-			}
-			log.Warnf("Reboot blocked: matching pods: %v", podNames)
-			return true
-		}
-	}
-	return false
-}
-
-func rebootBlocked(blockers ...RebootBlocker) bool {
-	for _, blocker := range blockers {
-		if blocker.isBlocked() {
-			return true
-		}
-	}
-	return false
-}
-
-func holding(lock *daemonsetlock.DaemonSetLock, metadata interface{}, isMultiLock bool) bool {
-	var holding bool
-	var err error
-	if isMultiLock {
-		holding, err = lock.TestMultiple()
-	} else {
-		holding, err = lock.Test(metadata)
-	}
-	if err != nil {
-		log.Fatalf("Error testing lock: %v", err)
-	}
-	if holding {
-		log.Infof("Holding lock")
-	}
-	return holding
-}
-
-func acquire(lock *daemonsetlock.DaemonSetLock, metadata interface{}, TTL time.Duration, maxOwners int) bool {
-	var holding bool
-	var holder string
-	var err error
-	if maxOwners > 1 {
-		var holders []string
-		holding, holders, err = lock.AcquireMultiple(metadata, TTL, maxOwners)
-		holder = strings.Join(holders, ",")
-	} else {
-		holding, holder, err = lock.Acquire(metadata, TTL)
-	}
-	switch {
-	case err != nil:
-		log.Fatalf("Error acquiring lock: %v", err)
-		return false
-	case !holding:
-		log.Warnf("Lock already held: %v", holder)
-		return false
-	default:
-		log.Infof("Acquired reboot lock")
-		return true
-	}
-}
-
-func throttle(releaseDelay time.Duration) {
-	if releaseDelay > 0 {
-		log.Infof("Delaying lock release by %v", releaseDelay)
-		time.Sleep(releaseDelay)
-	}
-}
-
-func release(lock *daemonsetlock.DaemonSetLock, isMultiLock bool) {
-	log.Infof("Releasing lock")
-
-	var err error
-	if isMultiLock {
-		err = lock.ReleaseMultiple()
-	} else {
-		err = lock.Release()
-	}
-	if err != nil {
-		log.Fatalf("Error releasing lock: %v", err)
-	}
-}
-
 func drain(client *kubernetes.Clientset, node *v1.Node) error {
 	nodename := node.GetName()
 
@@ -556,9 +468,9 @@ func uncordon(client *kubernetes.Clientset, node *v1.Node) error {
 	return nil
 }
 
-func maintainRebootRequiredMetric(nodeID string, sentinelCommand []string) {
+func maintainRebootRequiredMetric(nodeID string, checker checkers.Checker) {
 	for {
-		if rebootRequired(sentinelCommand) {
+		if checker.RebootRequired() {
 			rebootRequiredGauge.WithLabelValues(nodeID).Set(1)
 		} else {
 			rebootRequiredGauge.WithLabelValues(nodeID).Set(0)
@@ -567,11 +479,6 @@ func maintainRebootRequiredMetric(nodeID string, sentinelCommand []string) {
 	}
 }
 
-// nodeMeta is used to remember information across reboots
-type nodeMeta struct {
-	Unschedulable bool `json:"unschedulable"`
-}
-
 func addNodeAnnotations(client *kubernetes.Clientset, nodeID string, annotations map[string]string) error {
 	node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
 	if err != nil {
@@ -646,30 +553,23 @@ func updateNodeLabels(client *kubernetes.Clientset, node *v1.Node, labels []stri
 	}
 }
 
-func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []string, window *timewindow.TimeWindow, TTL time.Duration, releaseDelay time.Duration) {
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		log.Fatal(err)
-	}
+func rebootAsRequired(nodeID string, rebooter reboot.Rebooter, checker checkers.Checker, blockCheckers []blockers.RebootBlocker, window *timewindow.TimeWindow, lock daemonsetlock.Lock, client *kubernetes.Clientset) {
 
-	client, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	lock := daemonsetlock.New(client, nodeID, dsNamespace, dsName, lockAnnotation)
-
-	nodeMeta := nodeMeta{}
 	source := rand.NewSource(time.Now().UnixNano())
 	tick := delaytick.New(source, 1*time.Minute)
 	for range tick {
-		if holding(lock, &nodeMeta, concurrency > 1) {
+		holding, lockData, err := lock.Holding()
+		if err != nil {
+			log.Errorf("Error testing lock: %v", err)
+		}
+		if holding {
 			node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
 			if err != nil {
 				log.Errorf("Error retrieving node object via k8s API: %v", err)
 				continue
 			}
-			if !nodeMeta.Unschedulable {
+
+			if !lockData.Metadata.Unschedulable {
 				err = uncordon(client, node)
 				if err != nil {
 					log.Errorf("Unable to uncordon %s: %v, will continue to hold lock and retry uncordon", node.GetName(), err)
@@ -687,7 +587,7 @@ func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []str
 			// And (2) check if we previously annotated the node that it was in the process of being rebooted,
 			// And finally (3) if it has that annotation, to delete it.
 			// This indicates to other node tools running on the cluster that this node may be a candidate for maintenance
-			if annotateNodes && !rebootRequired(sentinelCommand) {
+			if annotateNodes && !checker.RebootRequired() {
 				if _, ok := node.Annotations[KuredRebootInProgressAnnotation]; ok {
 					err := deleteNodeAnnotation(client, nodeID, KuredRebootInProgressAnnotation)
 					if err != nil {
@@ -695,8 +595,12 @@ func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []str
 					}
 				}
 			}
-			throttle(releaseDelay)
-			release(lock, concurrency > 1)
+
+			err = lock.Release()
+			if err != nil {
+				log.Errorf("Error releasing lock, will retry: %v", err)
+				continue
+			}
 			break
 		} else {
 			break
@@ -706,16 +610,10 @@ func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []str
 	preferNoScheduleTaint := taints.New(client, nodeID, preferNoScheduleTaintName, v1.TaintEffectPreferNoSchedule)
 
 	// Remove taint immediately during startup to quickly allow scheduling again.
-	if !rebootRequired(sentinelCommand) {
+	if !checker.RebootRequired() {
 		preferNoScheduleTaint.Disable()
 	}
 
-	// instantiate prometheus client
-	promClient, err := alerts.NewPromClient(papi.Config{Address: prometheusURL})
-	if err != nil {
-		log.Fatal("Unable to create prometheus client: ", err)
-	}
-
 	source = rand.NewSource(time.Now().UnixNano())
 	tick = delaytick.New(source, period)
 	for range tick {
@@ -725,7 +623,7 @@ func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []str
 			continue
 		}
 
-		if !rebootRequired(sentinelCommand) {
+		if !checker.RebootRequired() {
 			log.Infof("Reboot not required")
 			preferNoScheduleTaint.Disable()
 			continue
@@ -735,7 +633,8 @@ func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []str
 		if err != nil {
 			log.Fatalf("Error retrieving node object via k8s API: %v", err)
 		}
-		nodeMeta.Unschedulable = node.Spec.Unschedulable
+
+		nodeMeta := daemonsetlock.NodeMeta{Unschedulable: node.Spec.Unschedulable}
 
 		var timeNowString string
 		if annotateNodes {
@@ -753,32 +652,39 @@ func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []str
 			}
 		}
 
-		var blockCheckers []RebootBlocker
-		if prometheusURL != "" {
-			blockCheckers = append(blockCheckers, PrometheusBlockingChecker{promClient: promClient, filter: alertFilter, firingOnly: alertFiringOnly, filterMatchOnly: alertFilterMatchOnly})
-		}
-		if podSelectors != nil {
-			blockCheckers = append(blockCheckers, KubernetesBlockingChecker{client: client, nodename: nodeID, filter: podSelectors})
-		}
-
 		var rebootRequiredBlockCondition string
-		if rebootBlocked(blockCheckers...) {
+		if blockers.RebootBlocked(blockCheckers...) {
 			rebootRequiredBlockCondition = ", but blocked at this time"
 			continue
 		}
 		log.Infof("Reboot required%s", rebootRequiredBlockCondition)
 
-		if !holding(lock, &nodeMeta, concurrency > 1) && !acquire(lock, &nodeMeta, TTL, concurrency) {
-			// Prefer to not schedule pods onto this node to avoid draing the same pod multiple times.
-			preferNoScheduleTaint.Enable()
-			continue
+		holding, _, err := lock.Holding()
+		if err != nil {
+			log.Errorf("Error testing lock: %v", err)
+		}
+
+		if !holding {
+			acquired, holder, err := lock.Acquire(nodeMeta)
+			if err != nil {
+				log.Errorf("Error acquiring lock: %v", err)
+			}
+			if !acquired {
+				log.Warnf("Lock already held: %v", holder)
+				// Prefer to not schedule pods onto this node to avoid draing the same pod multiple times.
+				preferNoScheduleTaint.Enable()
+				continue
+			}
 		}
 
 		err = drain(client, node)
 		if err != nil {
 			if !forceReboot {
 				log.Errorf("Unable to cordon or drain %s: %v, will release lock and retry cordon and drain before rebooting when lock is next acquired", node.GetName(), err)
-				release(lock, concurrency > 1)
+				err = lock.Release()
+				if err != nil {
+					log.Errorf("Error releasing lock: %v", err)
+				}
 				log.Infof("Performing a best-effort uncordon after failed cordon and drain")
 				uncordon(client, node)
 				continue
@@ -795,108 +701,15 @@ func rebootAsRequired(nodeID string, booter reboot.Reboot, sentinelCommand []str
 				log.Warnf("Error notifying: %v", err)
 			}
 		}
+		log.Infof("Triggering reboot for node %v", nodeID)
 
-		booter.Reboot()
+		err = rebooter.Reboot()
+		if err != nil {
+			log.Fatalf("Unable to reboot node: %v", err)
+		}
 		for {
 			log.Infof("Waiting for reboot")
 			time.Sleep(time.Minute)
 		}
 	}
 }
-
-// buildSentinelCommand creates the shell command line which will need wrapping to escape
-// the container boundaries
-func buildSentinelCommand(rebootSentinelFile string, rebootSentinelCommand string) []string {
-	if rebootSentinelCommand != "" {
-		cmd, err := shlex.Split(rebootSentinelCommand)
-		if err != nil {
-			log.Fatalf("Error parsing provided sentinel command: %v", err)
-		}
-		return cmd
-	}
-	return []string{"test", "-f", rebootSentinelFile}
-}
-
-// parseRebootCommand creates the shell command line which will need wrapping to escape
-// the container boundaries
-func parseRebootCommand(rebootCommand string) []string {
-	command, err := shlex.Split(rebootCommand)
-	if err != nil {
-		log.Fatalf("Error parsing provided reboot command: %v", err)
-	}
-	return command
-}
-
-func root(cmd *cobra.Command, args []string) {
-	if logFormat == "json" {
-		log.SetFormatter(&log.JSONFormatter{})
-	}
-
-	log.Infof("Kubernetes Reboot Daemon: %s", version)
-
-	if nodeID == "" {
-		log.Fatal("KURED_NODE_ID environment variable required")
-	}
-
-	window, err := timewindow.New(rebootDays, rebootStart, rebootEnd, timezone)
-	if err != nil {
-		log.Fatalf("Failed to build time window: %v", err)
-	}
-
-	sentinelCommand := buildSentinelCommand(rebootSentinelFile, rebootSentinelCommand)
-	restartCommand := parseRebootCommand(rebootCommand)
-
-	log.Infof("Node ID: %s", nodeID)
-	log.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation)
-	if lockTTL > 0 {
-		log.Infof("Lock TTL set, lock will expire after: %v", lockTTL)
-	} else {
-		log.Info("Lock TTL not set, lock will remain until being released")
-	}
-	if lockReleaseDelay > 0 {
-		log.Infof("Lock release delay set, lock release will be delayed by: %v", lockReleaseDelay)
-	} else {
-		log.Info("Lock release delay not set, lock will be released immediately after rebooting")
-	}
-	log.Infof("PreferNoSchedule taint: %s", preferNoScheduleTaintName)
-	log.Infof("Blocking Pod Selectors: %v", podSelectors)
-	log.Infof("Reboot schedule: %v", window)
-	log.Infof("Reboot check command: %s every %v", sentinelCommand, period)
-	log.Infof("Concurrency: %v", concurrency)
-	log.Infof("Reboot method: %s", rebootMethod)
-	if rebootCommand == MethodCommand {
-		log.Infof("Reboot command: %s", restartCommand)
-	} else {
-		log.Infof("Reboot signal: %v", rebootSignal)
-	}
-
-	if annotateNodes {
-		log.Infof("Will annotate nodes during kured reboot operations")
-	}
-
-	// To run those commands as it was the host, we'll use nsenter
-	// Relies on hostPID:true and privileged:true to enter host mount space
-	// PID set to 1, until we have a better discovery mechanism.
-	hostRestartCommand := buildHostCommand(1, restartCommand)
-
-	// Only wrap sentinel-command with nsenter, if a custom-command was configured, otherwise use the host-path mount
-	hostSentinelCommand := sentinelCommand
-	if rebootSentinelCommand != "" {
-		hostSentinelCommand = buildHostCommand(1, sentinelCommand)
-	}
-
-	var booter reboot.Reboot
-	if rebootMethod == MethodCommand {
-		booter = reboot.NewCommandReboot(nodeID, hostRestartCommand)
-	} else if rebootMethod == MethodSignal {
-		booter = reboot.NewSignalReboot(nodeID, rebootSignal)
-	} else {
-		log.Fatalf("Invalid reboot-method configured: %s", rebootMethod)
-	}
-
-	go rebootAsRequired(nodeID, booter, hostSentinelCommand, window, lockTTL, lockReleaseDelay)
-	go maintainRebootRequiredMetric(nodeID, hostSentinelCommand)
-
-	http.Handle("/metrics", promhttp.Handler())
-	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", metricsHost, metricsPort), nil))
-}

cmd/kured/main_test.go

@@ -3,61 +3,29 @@ package main
 import (
 	"reflect"
 	"testing"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/kubereboot/kured/pkg/alerts"
-	assert "gotest.tools/v3/assert"
-
-	papi "github.com/prometheus/client_golang/api"
 )
 
-type BlockingChecker struct {
-	blocking bool
-}
+func TestValidateNotificationURL(t *testing.T) {
 
-func (fbc BlockingChecker) isBlocked() bool {
-	return fbc.blocking
-}
-
-var _ RebootBlocker = BlockingChecker{}       // Verify that Type implements Interface.
-var _ RebootBlocker = (*BlockingChecker)(nil) // Verify that *Type implements Interface.
-
-func Test_flagCheck(t *testing.T) {
-	var cmd *cobra.Command
-	var args []string
-	slackHookURL = "https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"
-	expected := "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"
-	flagCheck(cmd, args)
-	if notifyURL != expected {
-		t.Errorf("Slack URL Parsing is wrong: expecting %s  but got %s\n", expected, notifyURL)
+	tests := []struct {
+		name         string
+		slackHookURL string
+		notifyURL    string
+		expected     string
+	}{
+		{"slackHookURL only works fine", "https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET", "", "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"},
+		{"slackHookURL and notify URL together only keeps notifyURL", "\"https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET\"", "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com", "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com"},
+		{"slackHookURL removes extraneous double quotes", "\"https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET\"", "", "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"},
+		{"slackHookURL removes extraneous single quotes", "'https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET'", "", "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"},
+		{"notifyURL removes extraneous double quotes", "", "\"teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com\"", "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com"},
+		{"notifyURL removes extraneous single quotes", "", "'teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com'", "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com"},
 	}
-
-	// validate that surrounding quotes are stripped
-	slackHookURL = "\"https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET\""
-	expected = "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"
-	flagCheck(cmd, args)
-	if notifyURL != expected {
-		t.Errorf("Slack URL Parsing is wrong: expecting %s  but got %s\n", expected, notifyURL)
-	}
-	slackHookURL = "'https://hooks.slack.com/services/BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET'"
-	expected = "slack://BLABLABA12345/IAM931A0VERY/COMPLICATED711854TOKEN1SET"
-	flagCheck(cmd, args)
-	if notifyURL != expected {
-		t.Errorf("Slack URL Parsing is wrong: expecting %s  but got %s\n", expected, notifyURL)
-	}
-	slackHookURL = ""
-	notifyURL = "\"teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com\""
-	expected = "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com"
-	flagCheck(cmd, args)
-	if notifyURL != expected {
-		t.Errorf("notifyURL Parsing is wrong: expecting %s  but got %s\n", expected, notifyURL)
-	}
-	notifyURL = "'teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com'"
-	expected = "teams://79b4XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX@acd8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/204cXXXXXXXXXXXXXXXXXXXXXXXXXXXX/a1f8XXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?host=XXXX.webhook.office.com"
-	flagCheck(cmd, args)
-	if notifyURL != expected {
-		t.Errorf("notifyURL Parsing is wrong: expecting %s  but got %s\n", expected, notifyURL)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := validateNotificationURL(tt.notifyURL, tt.slackHookURL); !reflect.DeepEqual(got, tt.expected) {
+				t.Errorf("validateNotificationURL() = %v, expected %v", got, tt.expected)
+			}
+		})
 	}
 }
 
@@ -106,205 +74,3 @@ func Test_stripQuotes(t *testing.T) {
 		})
 	}
 }
-
-func Test_rebootBlocked(t *testing.T) {
-	noCheckers := []RebootBlocker{}
-	nonblockingChecker := BlockingChecker{blocking: false}
-	blockingChecker := BlockingChecker{blocking: true}
-
-	// Instantiate a prometheusClient with a broken_url
-	promClient, err := alerts.NewPromClient(papi.Config{Address: "broken_url"})
-	if err != nil {
-		log.Fatal("Can't create prometheusClient: ", err)
-	}
-	brokenPrometheusClient := PrometheusBlockingChecker{promClient: promClient, filter: nil, firingOnly: false}
-
-	type args struct {
-		blockers []RebootBlocker
-	}
-	tests := []struct {
-		name string
-		args args
-		want bool
-	}{
-		{
-			name: "Do not block on no blocker defined",
-			args: args{blockers: noCheckers},
-			want: false,
-		},
-		{
-			name: "Ensure a blocker blocks",
-			args: args{blockers: []RebootBlocker{blockingChecker}},
-			want: true,
-		},
-		{
-			name: "Ensure a non-blocker doesn't block",
-			args: args{blockers: []RebootBlocker{nonblockingChecker}},
-			want: false,
-		},
-		{
-			name: "Ensure one blocker is enough to block",
-			args: args{blockers: []RebootBlocker{nonblockingChecker, blockingChecker}},
-			want: true,
-		},
-		{
-			name: "Do block on error contacting prometheus API",
-			args: args{blockers: []RebootBlocker{brokenPrometheusClient}},
-			want: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := rebootBlocked(tt.args.blockers...); got != tt.want {
-				t.Errorf("rebootBlocked() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_buildHostCommand(t *testing.T) {
-	type args struct {
-		pid     int
-		command []string
-	}
-	tests := []struct {
-		name string
-		args args
-		want []string
-	}{
-		{
-			name: "Ensure command will run with nsenter",
-			args: args{pid: 1, command: []string{"ls", "-Fal"}},
-			want: []string{"/usr/bin/nsenter", "-m/proc/1/ns/mnt", "--", "ls", "-Fal"},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := buildHostCommand(tt.args.pid, tt.args.command); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("buildHostCommand() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_buildSentinelCommand(t *testing.T) {
-	type args struct {
-		rebootSentinelFile    string
-		rebootSentinelCommand string
-	}
-	tests := []struct {
-		name string
-		args args
-		want []string
-	}{
-		{
-			name: "Ensure a sentinelFile generates a shell 'test' command with the right file",
-			args: args{
-				rebootSentinelFile:    "/test1",
-				rebootSentinelCommand: "",
-			},
-			want: []string{"test", "-f", "/test1"},
-		},
-		{
-			name: "Ensure a sentinelCommand has priority over a sentinelFile if both are provided (because sentinelFile is always provided)",
-			args: args{
-				rebootSentinelFile:    "/test1",
-				rebootSentinelCommand: "/sbin/reboot-required -r",
-			},
-			want: []string{"/sbin/reboot-required", "-r"},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := buildSentinelCommand(tt.args.rebootSentinelFile, tt.args.rebootSentinelCommand); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("buildSentinelCommand() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_parseRebootCommand(t *testing.T) {
-	type args struct {
-		rebootCommand string
-	}
-	tests := []struct {
-		name string
-		args args
-		want []string
-	}{
-		{
-			name: "Ensure a reboot command is properly parsed",
-			args: args{
-				rebootCommand: "/sbin/systemctl reboot",
-			},
-			want: []string{"/sbin/systemctl", "reboot"},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := parseRebootCommand(tt.args.rebootCommand); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("parseRebootCommand() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_rebootRequired(t *testing.T) {
-	type args struct {
-		sentinelCommand []string
-	}
-	tests := []struct {
-		name string
-		args args
-		want bool
-	}{
-		{
-			name: "Ensure rc = 0 means reboot required",
-			args: args{
-				sentinelCommand: []string{"true"},
-			},
-			want: true,
-		},
-		{
-			name: "Ensure rc != 0 means reboot NOT required",
-			args: args{
-				sentinelCommand: []string{"false"},
-			},
-			want: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := rebootRequired(tt.args.sentinelCommand); got != tt.want {
-				t.Errorf("rebootRequired() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_rebootRequired_fatals(t *testing.T) {
-	cases := []struct {
-		param       []string
-		expectFatal bool
-	}{
-		{
-			param:       []string{"true"},
-			expectFatal: false,
-		},
-		{
-			param:       []string{"./babar"},
-			expectFatal: true,
-		},
-	}
-
-	defer func() { log.StandardLogger().ExitFunc = nil }()
-	var fatal bool
-	log.StandardLogger().ExitFunc = func(int) { fatal = true }
-
-	for _, c := range cases {
-		fatal = false
-		rebootRequired(c.param)
-		assert.Equal(t, c.expectFatal, fatal)
-	}
-
-}

cmd/kured/pflags.go

@@ -5,14 +5,14 @@ import (
 )
 
 type regexpValue struct {
-	value **regexp.Regexp
+	*regexp.Regexp
 }
 
 func (rev *regexpValue) String() string {
-	if *rev.value == nil {
+	if rev.Regexp == nil {
 		return ""
 	}
-	return (*rev.value).String()
+	return rev.Regexp.String()
 }
 
 func (rev *regexpValue) Set(s string) error {
@@ -20,12 +20,11 @@ func (rev *regexpValue) Set(s string) error {
 	if err != nil {
 		return err
 	}
-
-	*rev.value = value
-
+	rev.Regexp = value
 	return nil
 }
 
+// Type method returns the type of the flag as a string
 func (rev *regexpValue) Type() string {
-	return "regexp.Regexp"
+	return "regexp"
 }

go.mod

@@ -1,41 +1,32 @@
 module github.com/kubereboot/kured
 
-go 1.21
-
-replace golang.org/x/net => golang.org/x/net v0.23.0
-
-replace github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 v3.10.2
+go 1.22.12
 
 require (
 	github.com/containrrr/shoutrrr v0.8.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
-	github.com/google/uuid v1.4.0 // indirect
-	github.com/prometheus/client_golang v1.19.1
-	github.com/prometheus/common v0.55.0
+	github.com/prometheus/client_golang v1.21.0
+	github.com/prometheus/common v0.62.0
 	github.com/sirupsen/logrus v1.9.3
-	github.com/spf13/cobra v1.8.1
-	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.19.0
-	github.com/stretchr/testify v1.9.0
-	gotest.tools/v3 v3.5.1
-	k8s.io/api v0.29.7
-	k8s.io/apimachinery v0.29.7
-	k8s.io/client-go v0.29.7
-	k8s.io/kubectl v0.29.7
+	github.com/spf13/pflag v1.0.6
+	github.com/stretchr/testify v1.10.0
+	k8s.io/api v0.30.10
+	k8s.io/apimachinery v0.30.10
+	k8s.io/client-go v0.30.10
+	k8s.io/kubectl v0.30.10
 )
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/fatih/color v1.15.0 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
@@ -45,22 +36,20 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.4.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
-	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -68,39 +57,30 @@ require (
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/sagikazarmark/locafero v0.4.0 // indirect
-	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/sourcegraph/conc v0.3.0 // indirect
-	github.com/spf13/afero v1.11.0 // indirect
-	github.com/spf13/cast v1.6.0 // indirect
-	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
-	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/term v0.18.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/oauth2 v0.24.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/protobuf v1.36.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/cli-runtime v0.29.7 // indirect
-	k8s.io/component-base v0.29.7 // indirect
-	k8s.io/klog/v2 v2.110.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/cli-runtime v0.30.10 // indirect
+	k8s.io/component-base v0.30.10 // indirect
+	k8s.io/klog/v2 v2.120.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect

go.sum

@@ -9,8 +9,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
 github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -27,8 +27,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE=
-github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
+github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
@@ -37,13 +37,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwC
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
-github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
-github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
-github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
-github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
-github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
@@ -95,8 +90,6 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -111,6 +104,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -118,10 +113,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
-github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
-github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -131,8 +126,6 @@ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPn
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
 github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
-github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
-github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA=
@@ -150,12 +143,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
-github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
-github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
-github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
-github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
-github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
+github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
+github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE=
+github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -163,39 +154,28 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
-github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA=
+github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
-github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
-github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
+github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
-github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
-github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
-github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
-github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
-github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
-github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
-github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
-github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
-github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -206,70 +186,60 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
-github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
 github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
-go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
-go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
-golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
+golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -280,8 +250,6 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -304,41 +272,37 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
-google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
+google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
-gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
-gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.29.7 h1:Q2/thp7YYESgy0MGzxT9RvA/6doLJHBXSFH8GGLxSbc=
-k8s.io/api v0.29.7/go.mod h1:mPimdbyuIjwoLtBEVIGVUYb4BKOE+44XHt/n4IqKsLA=
-k8s.io/apimachinery v0.29.7 h1:ICXzya58Q7hyEEfnTrbmdfX1n1schSepX2KUfC2/ykc=
-k8s.io/apimachinery v0.29.7/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y=
-k8s.io/cli-runtime v0.29.7 h1:6IxyxaIm3N31+PKXb1K7Tpf+100mm9hd9HMMYWMH2QE=
-k8s.io/cli-runtime v0.29.7/go.mod h1:0pcclC4k3rkzYNAvw3zeiPNtg8Buv0orK+5MuhEKFBU=
-k8s.io/client-go v0.29.7 h1:vTtiFrGBKlcBhxaeZC4eDrqui1e108nsTyue/KU63IY=
-k8s.io/client-go v0.29.7/go.mod h1:69BvVqdRozgR/9TP45u/oO0tfrdbP+I8RqrcCJQshzg=
-k8s.io/component-base v0.29.7 h1:zXLJvZjvvDWdYmZCwZYk95E1Fd2oRXUz71mQukkRk5I=
-k8s.io/component-base v0.29.7/go.mod h1:ddLTpIrjazaRI1EG83M41GNcYEAdskuQmx4JOOSXCOg=
-k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
-k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
-k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
-k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
-k8s.io/kubectl v0.29.7 h1:D+Jheug9M++zlt67cROZgxaIjrDdLqp9jkW/EYrXAoM=
-k8s.io/kubectl v0.29.7/go.mod h1:VOEJkcfKTO/X8xSSB6d2JXP/Qni6xtjuI3CUP52T9bM=
+k8s.io/api v0.30.10 h1:2YvzRF/BELgCvxbQqFKaan5hnj2+y7JOuqu2WpVk3gg=
+k8s.io/api v0.30.10/go.mod h1:Hyz3ZuK7jVLJBUFvwzDSGwxHuDdsrGs5RzF16wfHIn4=
+k8s.io/apimachinery v0.30.10 h1:UflKuJeSSArttm05wjYP0GwpTlvjnMbDKFn6F7rKkKU=
+k8s.io/apimachinery v0.30.10/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
+k8s.io/cli-runtime v0.30.10 h1:tSvl4yv7VgmZfgZcdjmv8KGD/Bota68pk9ywo1c/mcc=
+k8s.io/cli-runtime v0.30.10/go.mod h1:W9/mXoCYtFj/5dFtcUFTLJwhn8jZ6791HAFC8kWpM0s=
+k8s.io/client-go v0.30.10 h1:C0oWM82QMvosIl/IdJhWfTUb7rIxM52rNSutFBknAVY=
+k8s.io/client-go v0.30.10/go.mod h1:OfTvt0yuo8VpMViOsgvYQb+tMJQLNWVBqXWkzdFXSq4=
+k8s.io/component-base v0.30.10 h1:UJi0vTnTvtwWnVHcQeV1hzansnvTSKzFfMxtYAa8/GY=
+k8s.io/component-base v0.30.10/go.mod h1:q+6CkRDb/JOlqEpDzmuprysj4R/b/zzQO5vVBRynYQA=
+k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
+k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
+k8s.io/kubectl v0.30.10 h1:d/dsbA/JARUcL5wvRGqnjEvsBv7ageTj08PUCIcHwpE=
+k8s.io/kubectl v0.30.10/go.mod h1:2Okr39i+LHeK4QinNqy+IGivw8PCUcXIpfSUiZP8Llk=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=

internal/validators.go

@@ -0,0 +1,35 @@
+package internal
+
+import (
+	"fmt"
+	"github.com/kubereboot/kured/pkg/checkers"
+	"github.com/kubereboot/kured/pkg/reboot"
+	log "github.com/sirupsen/logrus"
+)
+
+// NewRebooter validates the rebootMethod, rebootCommand, and rebootSignal input,
+// then chains to the right constructor.
+func NewRebooter(rebootMethod string, rebootCommand string, rebootSignal int) (reboot.Rebooter, error) {
+	switch {
+	case rebootMethod == "command":
+		log.Infof("Reboot command: %s", rebootCommand)
+		return reboot.NewCommandRebooter(rebootCommand)
+	case rebootMethod == "signal":
+		log.Infof("Reboot signal: %d", rebootSignal)
+		return reboot.NewSignalRebooter(rebootSignal)
+	default:
+		return nil, fmt.Errorf("invalid reboot-method configured %s, expected signal or command", rebootMethod)
+	}
+}
+
+// NewRebootChecker validates the rebootSentinelCommand, rebootSentinelFile input,
+// then chains to the right constructor.
+func NewRebootChecker(rebootSentinelCommand string, rebootSentinelFile string) (checkers.Checker, error) {
+	// An override of rebootSentinelCommand means a privileged command
+	if rebootSentinelCommand != "" {
+		log.Infof("Sentinel checker is (privileged) user provided command: %s", rebootSentinelCommand)
+		return checkers.NewCommandChecker(rebootSentinelCommand, 1, true)
+	}
+	log.Infof("Sentinel checker is (unprivileged) testing for the presence of: %s", rebootSentinelFile)
+	return checkers.NewFileRebootChecker(rebootSentinelFile)
+}

kured-ds-signal.yaml

@@ -38,7 +38,7 @@ spec:
         - name: kured
           # If you find yourself here wondering why there is no
           # :latest tag on Docker Hub,see the FAQ in the README
-          image: ghcr.io/kubereboot/kured:1.16.0
+          image: ghcr.io/kubereboot/kured:1.17.1
           imagePullPolicy: IfNotPresent
           securityContext:
             privileged: false # Give permission to nsenter /proc/1/ns/mnt

kured-ds.yaml

@@ -38,7 +38,7 @@ spec:
         - name: kured
           # If you find yourself here wondering why there is no
           # :latest tag on Docker Hub,see the FAQ in the README
-          image: ghcr.io/kubereboot/kured:1.16.0
+          image: ghcr.io/kubereboot/kured:1.17.1
           imagePullPolicy: IfNotPresent
           securityContext:
             privileged: true # Give permission to nsenter /proc/1/ns/mnt

pkg/alerts/prometheus.go

@@ -1,77 +0,0 @@
-package alerts
-
-import (
-	"context"
-	"fmt"
-	"regexp"
-	"sort"
-	"time"
-
-	papi "github.com/prometheus/client_golang/api"
-	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
-	"github.com/prometheus/common/model"
-)
-
-// PromClient is a wrapper around the Prometheus Client interface and implements the api
-// This way, the PromClient can be instantiated with the configuration the Client needs, and
-// the ability to use the methods the api has, like Query and so on.
-type PromClient struct {
-	papi papi.Client
-	api  v1.API
-}
-
-// NewPromClient creates a new client to the Prometheus API.
-// It returns an error on any problem.
-func NewPromClient(conf papi.Config) (*PromClient, error) {
-	promClient, err := papi.NewClient(conf)
-	if err != nil {
-		return nil, err
-	}
-	client := PromClient{papi: promClient, api: v1.NewAPI(promClient)}
-	return &client, nil
-}
-
-// ActiveAlerts is a method of type PromClient, it returns a list of names of active alerts
-// (e.g. pending or firing), filtered by the supplied regexp or by the includeLabels query.
-// filter by regexp means when the regex finds the alert-name; the alert is exluded from the
-// block-list and will NOT block rebooting. query by includeLabel means,
-// if the query finds an alert, it will include it to the block-list and it WILL block rebooting.
-func (p *PromClient) ActiveAlerts(filter *regexp.Regexp, firingOnly, filterMatchOnly bool) ([]string, error) {
-
-	// get all alerts from prometheus
-	value, _, err := p.api.Query(context.Background(), "ALERTS", time.Now())
-	if err != nil {
-		return nil, err
-	}
-
-	if value.Type() == model.ValVector {
-		if vector, ok := value.(model.Vector); ok {
-			activeAlertSet := make(map[string]bool)
-			for _, sample := range vector {
-				if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 {
-					if matchesRegex(filter, string(alertName), filterMatchOnly) && (!firingOnly || sample.Metric["alertstate"] == "firing") {
-						activeAlertSet[string(alertName)] = true
-					}
-				}
-			}
-
-			var activeAlerts []string
-			for activeAlert := range activeAlertSet {
-				activeAlerts = append(activeAlerts, activeAlert)
-			}
-			sort.Strings(activeAlerts)
-
-			return activeAlerts, nil
-		}
-	}
-
-	return nil, fmt.Errorf("Unexpected value type: %v", value)
-}
-
-func matchesRegex(filter *regexp.Regexp, alertName string, filterMatchOnly bool) bool {
-	if filter == nil {
-		return true
-	}
-
-	return filter.MatchString(string(alertName)) == filterMatchOnly
-}

pkg/blockers/blockers.go

@@ -0,0 +1,18 @@
+package blockers
+
+// RebootBlocked checks that a single block Checker
+// will block the reboot or not.
+func RebootBlocked(blockers ...RebootBlocker) bool {
+	for _, blocker := range blockers {
+		if blocker.IsBlocked() {
+			return true
+		}
+	}
+	return false
+}
+
+// RebootBlocker interface should be implemented by types
+// to know if their instantiations should block a reboot
+type RebootBlocker interface {
+	IsBlocked() bool
+}

pkg/blockers/blockers_test.go

@@ -0,0 +1,65 @@
+package blockers
+
+import (
+	papi "github.com/prometheus/client_golang/api"
+	"testing"
+)
+
+type BlockingChecker struct {
+	blocking bool
+}
+
+func (fbc BlockingChecker) IsBlocked() bool {
+	return fbc.blocking
+}
+
+func Test_rebootBlocked(t *testing.T) {
+	noCheckers := []RebootBlocker{}
+	nonblockingChecker := BlockingChecker{blocking: false}
+	blockingChecker := BlockingChecker{blocking: true}
+
+	// Instantiate a prometheusClient with a broken_url
+	brokenPrometheusClient := NewPrometheusBlockingChecker(papi.Config{Address: "broken_url"}, nil, false, false)
+
+	type args struct {
+		blockers []RebootBlocker
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Do not block on no blocker defined",
+			args: args{blockers: noCheckers},
+			want: false,
+		},
+		{
+			name: "Ensure a blocker blocks",
+			args: args{blockers: []RebootBlocker{blockingChecker}},
+			want: true,
+		},
+		{
+			name: "Ensure a non-blocker doesn't block",
+			args: args{blockers: []RebootBlocker{nonblockingChecker}},
+			want: false,
+		},
+		{
+			name: "Ensure one blocker is enough to block",
+			args: args{blockers: []RebootBlocker{nonblockingChecker, blockingChecker}},
+			want: true,
+		},
+		{
+			name: "Do block on error contacting prometheus API",
+			args: args{blockers: []RebootBlocker{brokenPrometheusClient}},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := RebootBlocked(tt.args.blockers...); got != tt.want {
+				t.Errorf("rebootBlocked() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/blockers/kubernetespod.go

@@ -0,0 +1,61 @@
+package blockers
+
+import (
+	"context"
+	"fmt"
+	log "github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Compile-time checks to ensure the type implements the interface
+var (
+	_ RebootBlocker = (*KubernetesBlockingChecker)(nil)
+)
+
+// KubernetesBlockingChecker contains info for connecting
+// to k8s, and can give info about whether a reboot should be blocked
+type KubernetesBlockingChecker struct {
+	// client used to contact kubernetes API
+	client   *kubernetes.Clientset
+	nodeName string
+	// lised used to filter pods (podSelector)
+	filter []string
+}
+
+func NewKubernetesBlockingChecker(client *kubernetes.Clientset, nodename string, podSelectors []string) *KubernetesBlockingChecker {
+	return &KubernetesBlockingChecker{
+		client:   client,
+		nodeName: nodename,
+		filter:   podSelectors,
+	}
+}
+
+// IsBlocked for the KubernetesBlockingChecker will check if a pod, for the node, is preventing
+// the reboot. It will warn in the logs about blocking, but does not return an error.
+func (kb KubernetesBlockingChecker) IsBlocked() bool {
+	fieldSelector := fmt.Sprintf("spec.nodeName=%s,status.phase!=Succeeded,status.phase!=Failed,status.phase!=Unknown", kb.nodeName)
+	for _, labelSelector := range kb.filter {
+		podList, err := kb.client.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
+			LabelSelector: labelSelector,
+			FieldSelector: fieldSelector,
+			Limit:         10})
+		if err != nil {
+			log.Warnf("Reboot blocked: pod query error: %v", err)
+			return true
+		}
+
+		if len(podList.Items) > 0 {
+			podNames := make([]string, 0, len(podList.Items))
+			for _, pod := range podList.Items {
+				podNames = append(podNames, pod.Name)
+			}
+			if len(podList.Continue) > 0 {
+				podNames = append(podNames, "...")
+			}
+			log.Warnf("Reboot blocked: matching pods: %v", podNames)
+			return true
+		}
+	}
+	return false
+}

pkg/blockers/prometheus.go

@@ -0,0 +1,118 @@
+package blockers
+
+import (
+	"context"
+	"fmt"
+	papi "github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/common/model"
+	log "github.com/sirupsen/logrus"
+	"regexp"
+	"sort"
+	"time"
+)
+
+// Compile-time checks to ensure the type implements the interface
+var (
+	_ RebootBlocker = (*PrometheusBlockingChecker)(nil)
+)
+
+// PrometheusBlockingChecker contains info for connecting
+// to prometheus, and can give info about whether a reboot should be blocked
+type PrometheusBlockingChecker struct {
+	promConfig papi.Config
+	// regexp used to get alerts
+	filter *regexp.Regexp
+	// bool to indicate if only firing alerts should be considered
+	firingOnly bool
+	// bool to indicate that we're only blocking on alerts which match the filter
+	filterMatchOnly bool
+	// storing the promClient
+	promClient papi.Client
+}
+
+func NewPrometheusBlockingChecker(config papi.Config, alertFilter *regexp.Regexp, firingOnly bool, filterMatchOnly bool) PrometheusBlockingChecker {
+	promClient, _ := papi.NewClient(config)
+
+	return PrometheusBlockingChecker{
+		promConfig:      config,
+		filter:          alertFilter,
+		firingOnly:      firingOnly,
+		filterMatchOnly: filterMatchOnly,
+		promClient:      promClient,
+	}
+}
+
+// IsBlocked for the prometheus will check if there are active alerts matching
+// the arguments given into the PrometheusBlockingChecker which would actively
+// block the reboot.
+// As of today, no blocker information is shared as a return of the method,
+// and the information is simply logged.
+func (pb PrometheusBlockingChecker) IsBlocked() bool {
+	alertNames, err := pb.ActiveAlerts()
+	if err != nil {
+		log.Warnf("Reboot blocked: prometheus query error: %v", err)
+		return true
+	}
+	count := len(alertNames)
+	if count > 10 {
+		alertNames = append(alertNames[:10], "...")
+	}
+	if count > 0 {
+		log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
+		return true
+	}
+	return false
+}
+
+// MetricLabel is used to give a fancier name
+// than the type to the label for rebootBlockedCounter
+func (pb PrometheusBlockingChecker) MetricLabel() string {
+	return "prometheus"
+}
+
+// ActiveAlerts is a method of type promClient, it returns a list of names of active alerts
+// (e.g. pending or firing), filtered by the supplied regexp or by the includeLabels query.
+// filter by regexp means when the regexp finds the alert-name; the alert is excluded from the
+// block-list and will NOT block rebooting. query by includeLabel means,
+// if the query finds an alert, it will include it to the block-list, and it WILL block rebooting.
+func (pb PrometheusBlockingChecker) ActiveAlerts() ([]string, error) {
+	api := v1.NewAPI(pb.promClient)
+
+	// get all alerts from prometheus
+	value, _, err := api.Query(context.Background(), "ALERTS", time.Now())
+	if err != nil {
+		return nil, err
+	}
+
+	if value.Type() == model.ValVector {
+		if vector, ok := value.(model.Vector); ok {
+			activeAlertSet := make(map[string]bool)
+			for _, sample := range vector {
+				if alertName, isAlert := sample.Metric[model.AlertNameLabel]; isAlert && sample.Value != 0 {
+					if matchesRegex(pb.filter, string(alertName), pb.filterMatchOnly) && (!pb.firingOnly || sample.Metric["alertstate"] == "firing") {
+						activeAlertSet[string(alertName)] = true
+					}
+				}
+			}
+
+			var activeAlerts []string
+			for activeAlert := range activeAlertSet {
+				activeAlerts = append(activeAlerts, activeAlert)
+			}
+			sort.Strings(activeAlerts)
+
+			return activeAlerts, nil
+		}
+	}
+
+	return nil, fmt.Errorf("unexpected value type %v", value)
+}
+
+func matchesRegex(filter *regexp.Regexp, alertName string, filterMatchOnly bool) bool {
+	if filter == nil {
+		return true
+	}
+
+	return filter.MatchString(alertName) == filterMatchOnly
+}

pkg/blockers/prometheus_test.go

@@ -1,4 +1,4 @@
-package alerts
+package blockers
 
 import (
 	"log"
@@ -145,12 +145,9 @@ func TestActiveAlerts(t *testing.T) {
 			regex, _ := regexp.Compile(tc.rFilter)
 
 			// instantiate the prometheus client with the mockserver-address
-			p, err := NewPromClient(api.Config{Address: mockServer.URL})
-			if err != nil {
-				log.Fatal(err)
-			}
+			p := NewPrometheusBlockingChecker(api.Config{Address: mockServer.URL}, regex, tc.firingOnly, tc.filterMatchOnly)
 
-			result, err := p.ActiveAlerts(regex, tc.firingOnly, tc.filterMatchOnly)
+			result, err := p.ActiveAlerts()
 			if err != nil {
 				log.Fatal(err)
 			}

pkg/checkers/checker.go

@@ -0,0 +1,109 @@
+package checkers
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/google/shlex"
+	log "github.com/sirupsen/logrus"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// Checker is the standard interface to use to check
+// if a reboot is required. Its types must implement a
+// CheckRebootRequired method which returns a single boolean
+// clarifying whether a reboot is expected or not.
+type Checker interface {
+	RebootRequired() bool
+}
+
+// FileRebootChecker is the default reboot checker.
+// It is unprivileged, and tests the presence of a files
+type FileRebootChecker struct {
+	FilePath string
+}
+
+// RebootRequired checks the file presence
+// needs refactoring to also return an error, instead of leaking it inside the code.
+// This needs refactoring to get rid of NewCommand
+// This needs refactoring to only contain file location, instead of CheckCommand
+func (rc FileRebootChecker) RebootRequired() bool {
+	if _, err := os.Stat(rc.FilePath); err == nil {
+		return true
+	}
+	return false
+}
+
+// NewFileRebootChecker is the constructor for the file based reboot checker
+// TODO: Add extra input validation on filePath string here
+func NewFileRebootChecker(filePath string) (*FileRebootChecker, error) {
+	return &FileRebootChecker{
+		FilePath: filePath,
+	}, nil
+}
+
+// CommandChecker is using a custom command to check
+// if a reboot is required. There are two modes of behaviour,
+// if Privileged is granted, the NamespacePid is used to nsenter
+// the given PID's namespace.
+type CommandChecker struct {
+	CheckCommand []string
+	NamespacePid int
+	Privileged   bool
+}
+
+// RebootRequired for CommandChecker runs a command without returning
+// any eventual error. This should be later refactored to return the errors,
+// instead of logging and fataling them here.
+func (rc CommandChecker) RebootRequired() bool {
+	bufStdout := new(bytes.Buffer)
+	bufStderr := new(bytes.Buffer)
+	cmd := exec.Command(rc.CheckCommand[0], rc.CheckCommand[1:]...)
+	cmd.Stdout = bufStdout
+	cmd.Stderr = bufStderr
+
+	if err := cmd.Run(); err != nil {
+		switch err := err.(type) {
+		case *exec.ExitError:
+			// We assume a non-zero exit code means 'reboot not required', but of course
+			// the user could have misconfigured the sentinel command or something else
+			// went wrong during its execution. In that case, not entering a reboot loop
+			// is the right thing to do, and we are logging stdout/stderr of the command
+			// so it should be obvious what is wrong.
+			if cmd.ProcessState.ExitCode() != 1 {
+				log.Warn(fmt.Sprintf("sentinel command ended with unexpected exit code: %v", cmd.ProcessState.ExitCode()), "cmd", strings.Join(cmd.Args, " "), "stdout", bufStdout.String(), "stderr", bufStderr.String())
+			}
+			return false
+		default:
+			// Something was grossly misconfigured, such as the command path being wrong.
+			log.Fatal(fmt.Sprintf("Error invoking sentinel command: %v", err), "cmd", strings.Join(cmd.Args, " "), "stdout", bufStdout.String(), "stderr", bufStderr.String())
+		}
+	}
+	log.Info("checking if reboot is required", "cmd", strings.Join(cmd.Args, " "), "stdout", bufStdout.String(), "stderr", bufStderr.String())
+	return true
+}
+
+// NewCommandChecker is the constructor for the commandChecker, and by default
+// runs new commands in a privileged fashion.
+// Privileged means wrapping the command with nsenter.
+// It allows to run a command from systemd's namespace for example (pid 1)
+// This relies on hostPID:true and privileged:true to enter host mount space
+// For info, rancher based need different pid, which should be user given.
+// until we have a better discovery mechanism.
+func NewCommandChecker(sentinelCommand string, pid int, privileged bool) (*CommandChecker, error) {
+	var cmd []string
+	if privileged {
+		cmd = append(cmd, "/usr/bin/nsenter", fmt.Sprintf("-m/proc/%d/ns/mnt", pid), "--")
+	}
+	parsedCommand, err := shlex.Split(sentinelCommand)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing provided sentinel command: %v", err)
+	}
+	cmd = append(cmd, parsedCommand...)
+	return &CommandChecker{
+		CheckCommand: cmd,
+		NamespacePid: pid,
+		Privileged:   privileged,
+	}, nil
+}

pkg/checkers/checker_test.go

@@ -0,0 +1,87 @@
+package checkers
+
+import (
+	log "github.com/sirupsen/logrus"
+	"reflect"
+	"testing"
+)
+
+func Test_nsEntering(t *testing.T) {
+	type args struct {
+		pid        int
+		command    string
+		privileged bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "Ensure command will run with nsenter",
+			args: args{pid: 1, command: "ls -Fal", privileged: true},
+			want: []string{"/usr/bin/nsenter", "-m/proc/1/ns/mnt", "--", "ls", "-Fal"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cc, _ := NewCommandChecker(tt.args.command, tt.args.pid, tt.args.privileged)
+			if !reflect.DeepEqual(cc.CheckCommand, tt.want) {
+				t.Errorf("command parsed as %v, want %v", cc.CheckCommand, tt.want)
+			}
+		})
+	}
+}
+
+func Test_rebootRequired(t *testing.T) {
+	type args struct {
+		sentinelCommand []string
+	}
+	tests := []struct {
+		name   string
+		args   args
+		want   bool
+		fatals bool
+	}{
+		{
+			name: "Ensure rc = 0 means reboot required",
+			args: args{
+				sentinelCommand: []string{"true"},
+			},
+			want:   true,
+			fatals: false,
+		},
+		{
+			name: "Ensure rc != 0 means reboot NOT required",
+			args: args{
+				sentinelCommand: []string{"false"},
+			},
+			want:   false,
+			fatals: false,
+		},
+		{
+			name: "Ensure a wrong command fatals",
+			args: args{
+				sentinelCommand: []string{"./babar"},
+			},
+			want:   true,
+			fatals: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			defer func() { log.StandardLogger().ExitFunc = nil }()
+			fatal := false
+			log.StandardLogger().ExitFunc = func(int) { fatal = true }
+
+			a := CommandChecker{CheckCommand: tt.args.sentinelCommand, NamespacePid: 1, Privileged: false}
+
+			if got := a.RebootRequired(); got != tt.want {
+				t.Errorf("rebootRequired() = %v, want %v", got, tt.want)
+			}
+			if tt.fatals != fatal {
+				t.Errorf("fatal flag is %v, want fatal %v", fatal, tt.fatals)
+			}
+		})
+	}
+}

pkg/daemonsetlock/daemonsetlock.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	log "github.com/sirupsen/logrus"
+	"strings"
 	"time"
 
 	v1 "k8s.io/api/apps/v1"
@@ -18,6 +20,21 @@ const (
 	k8sAPICallRetryTimeout = 5 * time.Minute // How long to wait until we determine that the k8s API is definitively unavailable
 )
 
+type Lock interface {
+	Acquire(NodeMeta) (bool, string, error)
+	Release() error
+	Holding() (bool, LockAnnotationValue, error)
+}
+
+type GenericLock struct {
+	TTL          time.Duration
+	releaseDelay time.Duration
+}
+
+type NodeMeta struct {
+	Unschedulable bool `json:"unschedulable"`
+}
+
 // DaemonSetLock holds all necessary information to do actions
 // on the kured ds which holds lock info through annotations.
 type DaemonSetLock struct {
@@ -28,25 +45,90 @@ type DaemonSetLock struct {
 	annotation string
 }
 
-type lockAnnotationValue struct {
+// DaemonSetSingleLock holds all necessary information to do actions
+// on the kured ds which holds lock info through annotations.
+type DaemonSetSingleLock struct {
+	GenericLock
+	DaemonSetLock
+}
+
+// DaemonSetMultiLock holds all necessary information to do actions
+// on the kured ds which holds lock info through annotations, valid
+// for multiple nodes
+type DaemonSetMultiLock struct {
+	GenericLock
+	DaemonSetLock
+	maxOwners int
+}
+
+// LockAnnotationValue contains the lock data,
+// which allows persistence across reboots, particularily recording if the
+// node was already unschedulable before kured reboot.
+// To be modified when using another type of lock storage.
+type LockAnnotationValue struct {
 	NodeID   string        `json:"nodeID"`
-	Metadata interface{}   `json:"metadata,omitempty"`
+	Metadata NodeMeta      `json:"metadata,omitempty"`
 	Created  time.Time     `json:"created"`
 	TTL      time.Duration `json:"TTL"`
 }
 
 type multiLockAnnotationValue struct {
 	MaxOwners       int                   `json:"maxOwners"`
-	LockAnnotations []lockAnnotationValue `json:"locks"`
+	LockAnnotations []LockAnnotationValue `json:"locks"`
 }
 
 // New creates a daemonsetLock object containing the necessary data for follow up k8s requests
-func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation string) *DaemonSetLock {
-	return &DaemonSetLock{client, nodeID, namespace, name, annotation}
+func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation string, TTL time.Duration, concurrency int, lockReleaseDelay time.Duration) Lock {
+	if concurrency > 1 {
+		return &DaemonSetMultiLock{
+			GenericLock: GenericLock{
+				TTL:          TTL,
+				releaseDelay: lockReleaseDelay,
+			},
+			DaemonSetLock: DaemonSetLock{
+				client:     client,
+				nodeID:     nodeID,
+				namespace:  namespace,
+				name:       name,
+				annotation: annotation,
+			},
+			maxOwners: concurrency,
+		}
+	} else {
+		return &DaemonSetSingleLock{
+			GenericLock: GenericLock{
+				TTL:          TTL,
+				releaseDelay: lockReleaseDelay,
+			},
+			DaemonSetLock: DaemonSetLock{
+				client:     client,
+				nodeID:     nodeID,
+				namespace:  namespace,
+				name:       name,
+				annotation: annotation,
+			},
+		}
+	}
+}
+
+// GetDaemonSet returns the named DaemonSet resource from the DaemonSetLock's configured client
+func (dsl *DaemonSetLock) GetDaemonSet(sleep, timeout time.Duration) (*v1.DaemonSet, error) {
+	var ds *v1.DaemonSet
+	var lastError error
+	err := wait.PollUntilContextTimeout(context.Background(), sleep, timeout, true, func(ctx context.Context) (bool, error) {
+		if ds, lastError = dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(ctx, dsl.name, metav1.GetOptions{}); lastError != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %v", dsl.name, dsl.namespace, lastError)
+	}
+	return ds, nil
 }
 
 // Acquire attempts to annotate the kured daemonset with lock info from instantiated DaemonSetLock using client-go
-func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (bool, string, error) {
+func (dsl *DaemonSetSingleLock) Acquire(nodeMetadata NodeMeta) (bool, string, error) {
 	for {
 		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
@@ -55,7 +137,7 @@ func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (bool
 
 		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
 		if exists {
-			value := lockAnnotationValue{}
+			value := LockAnnotationValue{}
 			if err := json.Unmarshal([]byte(valueString), &value); err != nil {
 				return false, "", err
 			}
@@ -68,7 +150,7 @@ func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (bool
 		if ds.ObjectMeta.Annotations == nil {
 			ds.ObjectMeta.Annotations = make(map[string]string)
 		}
-		value := lockAnnotationValue{NodeID: dsl.nodeID, Metadata: metadata, Created: time.Now().UTC(), TTL: TTL}
+		value := LockAnnotationValue{NodeID: dsl.nodeID, Metadata: nodeMetadata, Created: time.Now().UTC(), TTL: dsl.TTL}
 		valueBytes, err := json.Marshal(&value)
 		if err != nil {
 			return false, "", err
@@ -89,49 +171,78 @@ func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (bool
 	}
 }
 
-// AcquireMultiple creates and annotates the daemonset with a multiple owner lock
-func (dsl *DaemonSetLock) AcquireMultiple(metadata interface{}, TTL time.Duration, maxOwners int) (bool, []string, error) {
+// Test attempts to check the kured daemonset lock status (existence, expiry) from instantiated DaemonSetLock using client-go
+func (dsl *DaemonSetSingleLock) Holding() (bool, LockAnnotationValue, error) {
+	var lockData LockAnnotationValue
+	ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
+	if err != nil {
+		return false, lockData, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
+	}
+
+	valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
+	if exists {
+		value := LockAnnotationValue{}
+		if err := json.Unmarshal([]byte(valueString), &value); err != nil {
+			return false, lockData, err
+		}
+
+		if !ttlExpired(value.Created, value.TTL) {
+			return value.NodeID == dsl.nodeID, value, nil
+		}
+	}
+
+	return false, lockData, nil
+}
+
+// Release attempts to remove the lock data from the kured ds annotations using client-go
+func (dsl *DaemonSetSingleLock) Release() error {
+	if dsl.releaseDelay > 0 {
+		log.Infof("Waiting %v before releasing lock", dsl.releaseDelay)
+		time.Sleep(dsl.releaseDelay)
+	}
 	for {
 		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
-			return false, []string{}, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
+			return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 		}
 
-		annotation := multiLockAnnotationValue{}
 		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
 		if exists {
-			if err := json.Unmarshal([]byte(valueString), &annotation); err != nil {
-				return false, []string{}, fmt.Errorf("error getting multi lock: %w", err)
+			value := LockAnnotationValue{}
+			if err := json.Unmarshal([]byte(valueString), &value); err != nil {
+				return err
 			}
+
+			if value.NodeID != dsl.nodeID {
+				return fmt.Errorf("not lock holder: %v", value.NodeID)
+			}
+		} else {
+			return fmt.Errorf("lock not held")
 		}
 
-		lockPossible, newAnnotation := dsl.canAcquireMultiple(annotation, metadata, TTL, maxOwners)
-		if !lockPossible {
-			return false, nodeIDsFromMultiLock(newAnnotation), nil
-		}
+		delete(ds.ObjectMeta.Annotations, dsl.annotation)
 
-		if ds.ObjectMeta.Annotations == nil {
-			ds.ObjectMeta.Annotations = make(map[string]string)
-		}
-		newAnnotationBytes, err := json.Marshal(&newAnnotation)
-		if err != nil {
-			return false, []string{}, fmt.Errorf("error marshalling new annotation lock: %w", err)
-		}
-		ds.ObjectMeta.Annotations[dsl.annotation] = string(newAnnotationBytes)
-
-		_, err = dsl.client.AppsV1().DaemonSets(dsl.namespace).Update(context.Background(), ds, metav1.UpdateOptions{})
+		_, err = dsl.client.AppsV1().DaemonSets(dsl.namespace).Update(context.TODO(), ds, metav1.UpdateOptions{})
 		if err != nil {
 			if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == metav1.StatusReasonConflict {
+				// Something else updated the resource between us reading and writing - try again soon
 				time.Sleep(time.Second)
 				continue
 			} else {
-				return false, []string{}, fmt.Errorf("error updating daemonset with multi lock: %w", err)
+				return err
 			}
 		}
-		return true, nodeIDsFromMultiLock(newAnnotation), nil
+		return nil
 	}
 }
 
+func ttlExpired(created time.Time, ttl time.Duration) bool {
+	if ttl > 0 && time.Since(created) >= ttl {
+		return true
+	}
+	return false
+}
+
 func nodeIDsFromMultiLock(annotation multiLockAnnotationValue) []string {
 	nodeIDs := make([]string, 0, len(annotation.LockAnnotations))
 	for _, nodeLock := range annotation.LockAnnotations {
@@ -140,7 +251,7 @@ func nodeIDsFromMultiLock(annotation multiLockAnnotationValue) []string {
 	return nodeIDs
 }
 
-func (dsl *DaemonSetLock) canAcquireMultiple(annotation multiLockAnnotationValue, metadata interface{}, TTL time.Duration, maxOwners int) (bool, multiLockAnnotationValue) {
+func (dsl *DaemonSetLock) canAcquireMultiple(annotation multiLockAnnotationValue, metadata NodeMeta, TTL time.Duration, maxOwners int) (bool, multiLockAnnotationValue) {
 	newAnnotation := multiLockAnnotationValue{MaxOwners: maxOwners}
 	freeSpace := false
 	if annotation.LockAnnotations == nil || len(annotation.LockAnnotations) < maxOwners {
@@ -162,7 +273,7 @@ func (dsl *DaemonSetLock) canAcquireMultiple(annotation multiLockAnnotationValue
 	if freeSpace {
 		newAnnotation.LockAnnotations = append(
 			newAnnotation.LockAnnotations,
-			lockAnnotationValue{
+			LockAnnotationValue{
 				NodeID:   dsl.nodeID,
 				Metadata: metadata,
 				Created:  time.Now().UTC(),
@@ -175,92 +286,80 @@ func (dsl *DaemonSetLock) canAcquireMultiple(annotation multiLockAnnotationValue
 	return false, multiLockAnnotationValue{}
 }
 
-// Test attempts to check the kured daemonset lock status (existence, expiry) from instantiated DaemonSetLock using client-go
-func (dsl *DaemonSetLock) Test(metadata interface{}) (bool, error) {
-	ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
-	if err != nil {
-		return false, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
-	}
-
-	valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
-	if exists {
-		value := lockAnnotationValue{Metadata: metadata}
-		if err := json.Unmarshal([]byte(valueString), &value); err != nil {
-			return false, err
+// Acquire creates and annotates the daemonset with a multiple owner lock
+func (dsl *DaemonSetMultiLock) Acquire(nodeMetaData NodeMeta) (bool, string, error) {
+	for {
+		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
+		if err != nil {
+			return false, "", fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 		}
 
-		if !ttlExpired(value.Created, value.TTL) {
-			return value.NodeID == dsl.nodeID, nil
+		annotation := multiLockAnnotationValue{}
+		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
+		if exists {
+			if err := json.Unmarshal([]byte(valueString), &annotation); err != nil {
+				return false, "", fmt.Errorf("error getting multi lock: %w", err)
+			}
 		}
-	}
 
-	return false, nil
+		lockPossible, newAnnotation := dsl.canAcquireMultiple(annotation, nodeMetaData, dsl.TTL, dsl.maxOwners)
+		if !lockPossible {
+			return false, strings.Join(nodeIDsFromMultiLock(newAnnotation), ","), nil
+		}
+
+		if ds.ObjectMeta.Annotations == nil {
+			ds.ObjectMeta.Annotations = make(map[string]string)
+		}
+		newAnnotationBytes, err := json.Marshal(&newAnnotation)
+		if err != nil {
+			return false, "", fmt.Errorf("error marshalling new annotation lock: %w", err)
+		}
+		ds.ObjectMeta.Annotations[dsl.annotation] = string(newAnnotationBytes)
+
+		_, err = dsl.client.AppsV1().DaemonSets(dsl.namespace).Update(context.Background(), ds, metav1.UpdateOptions{})
+		if err != nil {
+			if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == metav1.StatusReasonConflict {
+				time.Sleep(time.Second)
+				continue
+			} else {
+				return false, "", fmt.Errorf("error updating daemonset with multi lock: %w", err)
+			}
+		}
+		return true, strings.Join(nodeIDsFromMultiLock(newAnnotation), ","), nil
+	}
 }
 
 // TestMultiple attempts to check the kured daemonset lock status for multi locks
-func (dsl *DaemonSetLock) TestMultiple() (bool, error) {
+func (dsl *DaemonSetMultiLock) Holding() (bool, LockAnnotationValue, error) {
+	var lockdata LockAnnotationValue
 	ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 	if err != nil {
-		return false, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
+		return false, lockdata, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
 	}
 
 	valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
 	if exists {
 		value := multiLockAnnotationValue{}
 		if err := json.Unmarshal([]byte(valueString), &value); err != nil {
-			return false, err
+			return false, lockdata, err
 		}
 
 		for _, nodeLock := range value.LockAnnotations {
 			if nodeLock.NodeID == dsl.nodeID && !ttlExpired(nodeLock.Created, nodeLock.TTL) {
-				return true, nil
+				return true, nodeLock, nil
 			}
 		}
 	}
 
-	return false, nil
+	return false, lockdata, nil
 }
 
-// Release attempts to remove the lock data from the kured ds annotations using client-go
-func (dsl *DaemonSetLock) Release() error {
-	for {
-		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
-		if err != nil {
-			return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err)
-		}
-
-		valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation]
-		if exists {
-			value := lockAnnotationValue{}
-			if err := json.Unmarshal([]byte(valueString), &value); err != nil {
-				return err
-			}
-
-			if value.NodeID != dsl.nodeID {
-				return fmt.Errorf("Not lock holder: %v", value.NodeID)
-			}
-		} else {
-			return fmt.Errorf("Lock not held")
-		}
-
-		delete(ds.ObjectMeta.Annotations, dsl.annotation)
-
-		_, err = dsl.client.AppsV1().DaemonSets(dsl.namespace).Update(context.TODO(), ds, metav1.UpdateOptions{})
-		if err != nil {
-			if se, ok := err.(*errors.StatusError); ok && se.ErrStatus.Reason == metav1.StatusReasonConflict {
-				// Something else updated the resource between us reading and writing - try again soon
-				time.Sleep(time.Second)
-				continue
-			} else {
-				return err
-			}
-		}
-		return nil
+// Release attempts to remove the lock data for a single node from the multi node annotation
+func (dsl *DaemonSetMultiLock) Release() error {
+	if dsl.releaseDelay > 0 {
+		log.Infof("Waiting %v before releasing lock", dsl.releaseDelay)
+		time.Sleep(dsl.releaseDelay)
 	}
-}
-
-// ReleaseMultiple attempts to remove the lock data from the kured ds annotations using client-go
-func (dsl *DaemonSetLock) ReleaseMultiple() error {
 	for {
 		ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout)
 		if err != nil {
@@ -307,28 +406,3 @@ func (dsl *DaemonSetLock) ReleaseMultiple() error {
 		return nil
 	}
 }
-
-// GetDaemonSet returns the named DaemonSet resource from the DaemonSetLock's configured client
-func (dsl *DaemonSetLock) GetDaemonSet(sleep, timeout time.Duration) (*v1.DaemonSet, error) {
-	var ds *v1.DaemonSet
-	var lastError error
-	err := wait.PollImmediate(sleep, timeout, func() (bool, error) {
-		ctx, cancel := context.WithTimeout(context.Background(), timeout)
-		defer cancel()
-		if ds, lastError = dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(ctx, dsl.name, metav1.GetOptions{}); lastError != nil {
-			return false, nil
-		}
-		return true, nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("Timed out trying to get daemonset %s in namespace %s: %v", dsl.name, dsl.namespace, lastError)
-	}
-	return ds, nil
-}
-
-func ttlExpired(created time.Time, ttl time.Duration) bool {
-	if ttl > 0 && time.Since(created) >= ttl {
-		return true
-	}
-	return false
-}

pkg/daemonsetlock/daemonsetlock_test.go

@@ -66,7 +66,7 @@ func TestCanAcquireMultiple(t *testing.T) {
 			current:   multiLockAnnotationValue{},
 			desired: multiLockAnnotationValue{
 				MaxOwners: 2,
-				LockAnnotations: []lockAnnotationValue{
+				LockAnnotations: []LockAnnotationValue{
 					{NodeID: node1Name},
 				},
 			},
@@ -80,13 +80,13 @@ func TestCanAcquireMultiple(t *testing.T) {
 			maxOwners: 2,
 			current: multiLockAnnotationValue{
 				MaxOwners: 2,
-				LockAnnotations: []lockAnnotationValue{
+				LockAnnotations: []LockAnnotationValue{
 					{NodeID: node2Name},
 				},
 			},
 			desired: multiLockAnnotationValue{
 				MaxOwners: 2,
-				LockAnnotations: []lockAnnotationValue{
+				LockAnnotations: []LockAnnotationValue{
 					{NodeID: node1Name},
 					{NodeID: node2Name},
 				},
@@ -101,7 +101,7 @@ func TestCanAcquireMultiple(t *testing.T) {
 			maxOwners: 2,
 			current: multiLockAnnotationValue{
 				MaxOwners: 2,
-				LockAnnotations: []lockAnnotationValue{
+				LockAnnotations: []LockAnnotationValue{
 					{
 						NodeID:  node2Name,
 						Created: time.Now().UTC().Add(-1 * time.Minute),
@@ -116,7 +116,7 @@ func TestCanAcquireMultiple(t *testing.T) {
 			},
 			desired: multiLockAnnotationValue{
 				MaxOwners: 2,
-				LockAnnotations: []lockAnnotationValue{
+				LockAnnotations: []LockAnnotationValue{
 					{NodeID: node2Name},
 					{NodeID: node3Name},
 				},
@@ -131,7 +131,7 @@ func TestCanAcquireMultiple(t *testing.T) {
 			maxOwners: 2,
 			current: multiLockAnnotationValue{
 				MaxOwners: 2,
-				LockAnnotations: []lockAnnotationValue{
+				LockAnnotations: []LockAnnotationValue{
 					{
 						NodeID:  node2Name,
 						Created: time.Now().UTC().Add(-1 * time.Hour),
@@ -146,7 +146,7 @@ func TestCanAcquireMultiple(t *testing.T) {
 			},
 			desired: multiLockAnnotationValue{
 				MaxOwners: 2,
-				LockAnnotations: []lockAnnotationValue{
+				LockAnnotations: []LockAnnotationValue{
 					{NodeID: node1Name},
 					{NodeID: node3Name},
 				},
@@ -161,7 +161,7 @@ func TestCanAcquireMultiple(t *testing.T) {
 			maxOwners: 2,
 			current: multiLockAnnotationValue{
 				MaxOwners: 2,
-				LockAnnotations: []lockAnnotationValue{
+				LockAnnotations: []LockAnnotationValue{
 					{
 						NodeID:  node2Name,
 						Created: time.Now().UTC().Add(-1 * time.Hour),
@@ -176,17 +176,17 @@ func TestCanAcquireMultiple(t *testing.T) {
 			},
 			desired: multiLockAnnotationValue{
 				MaxOwners: 2,
-				LockAnnotations: []lockAnnotationValue{
+				LockAnnotations: []LockAnnotationValue{
 					{NodeID: node1Name},
 				},
 			},
 			lockPossible: true,
 		},
 	}
-
+	nm := NodeMeta{Unschedulable: false}
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			lockPossible, actual := testCase.daemonSetLock.canAcquireMultiple(testCase.current, struct{}{}, time.Minute, testCase.maxOwners)
+			lockPossible, actual := testCase.daemonSetLock.canAcquireMultiple(testCase.current, nm, time.Minute, testCase.maxOwners)
 			if lockPossible != testCase.lockPossible {
 				t.Fatalf(
 					"unexpected result for lock possible (got %t expected %t new annotation %v",

pkg/reboot/command.go

@@ -1,25 +1,48 @@
 package reboot
 
 import (
-	"github.com/kubereboot/kured/pkg/util"
+	"bytes"
+	"fmt"
+	"github.com/google/shlex"
 	log "github.com/sirupsen/logrus"
+	"os/exec"
+	"strings"
 )
 
-// CommandRebootMethod holds context-information for a command reboot.
-type CommandRebootMethod struct {
-	nodeID        string
-	rebootCommand []string
+// CommandRebooter holds context-information for a reboot with command
+type CommandRebooter struct {
+	RebootCommand []string
 }
 
-// NewCommandReboot creates a new command-rebooter which needs full privileges on the host.
-func NewCommandReboot(nodeID string, rebootCommand []string) *CommandRebootMethod {
-	return &CommandRebootMethod{nodeID: nodeID, rebootCommand: rebootCommand}
-}
+// Reboot triggers the reboot command
+func (c CommandRebooter) Reboot() error {
+	log.Infof("Invoking command: %s", c.RebootCommand)
 
-// Reboot triggers the command-reboot.
-func (c *CommandRebootMethod) Reboot() {
-	log.Infof("Running command: %s for node: %s", c.rebootCommand, c.nodeID)
-	if err := util.NewCommand(c.rebootCommand[0], c.rebootCommand[1:]...).Run(); err != nil {
-		log.Fatalf("Error invoking reboot command: %v", err)
+	bufStdout := new(bytes.Buffer)
+	bufStderr := new(bytes.Buffer)
+	cmd := exec.Command(c.RebootCommand[0], c.RebootCommand[1:]...)
+	cmd.Stdout = bufStdout
+	cmd.Stderr = bufStderr
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("error invoking reboot command %s: %v (stdout: %v, stderr: %v)", c.RebootCommand, err, bufStdout.String(), bufStderr.String())
 	}
+	log.Info("Invoked reboot command", "cmd", strings.Join(cmd.Args, " "), "stdout", bufStdout.String(), "stderr", bufStderr.String())
+	return nil
+}
+
+// NewCommandRebooter is the constructor to create a CommandRebooter from a string not
+// yet shell lexed. You can skip this constructor if you parse the data correctly first
+// when instantiating a CommandRebooter instance.
+func NewCommandRebooter(rebootCommand string) (*CommandRebooter, error) {
+	if rebootCommand == "" {
+		return nil, fmt.Errorf("no reboot command specified")
+	}
+	cmd := []string{"/usr/bin/nsenter", fmt.Sprintf("-m/proc/%d/ns/mnt", 1), "--"}
+	parsedCommand, err := shlex.Split(rebootCommand)
+	if err != nil {
+		return nil, fmt.Errorf("error %v when parsing reboot command %s", err, rebootCommand)
+	}
+	cmd = append(cmd, parsedCommand...)
+	return &CommandRebooter{RebootCommand: cmd}, nil
 }

pkg/reboot/command_test.go

@@ -0,0 +1,43 @@
+package reboot
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestNewCommandRebooter(t *testing.T) {
+	type args struct {
+		rebootCommand string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *CommandRebooter
+		wantErr bool
+	}{
+		{
+			name:    "Ensure command is nsenter wrapped",
+			args:    args{"ls -Fal"},
+			want:    &CommandRebooter{RebootCommand: []string{"/usr/bin/nsenter", "-m/proc/1/ns/mnt", "--", "ls", "-Fal"}},
+			wantErr: false,
+		},
+		{
+			name:    "Ensure empty command is erroring",
+			args:    args{""},
+			want:    nil,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := NewCommandRebooter(tt.args.rebootCommand)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("NewCommandRebooter() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("NewCommandRebooter() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/reboot/reboot.go

@@ -1,6 +1,9 @@
 package reboot
 
-// Reboot interface defines the Reboot function to be implemented.
-type Reboot interface {
-	Reboot()
+// Rebooter is the standard interface to use to execute
+// the reboot, after it has been considered as necessary.
+// The Reboot method does not expect any return, yet should
+// most likely be refactored in the future to return an error
+type Rebooter interface {
+	Reboot() error
 }

pkg/reboot/signal.go

@@ -1,34 +1,37 @@
 package reboot
 
 import (
+	"fmt"
 	"os"
 	"syscall"
-
-	log "github.com/sirupsen/logrus"
 )
 
-// SignalRebootMethod holds context-information for a signal reboot.
-type SignalRebootMethod struct {
-	nodeID string
-	signal int
+// SignalRebooter holds context-information for a signal reboot.
+type SignalRebooter struct {
+	Signal int
 }
 
-// NewSignalReboot creates a new signal-rebooter which can run unprivileged.
-func NewSignalReboot(nodeID string, signal int) *SignalRebootMethod {
-	return &SignalRebootMethod{nodeID: nodeID, signal: signal}
-}
-
-// Reboot triggers the signal-reboot.
-func (c *SignalRebootMethod) Reboot() {
-	log.Infof("Emit reboot-signal for node: %s", c.nodeID)
-
+// Reboot triggers the reboot signal
+func (c SignalRebooter) Reboot() error {
 	process, err := os.FindProcess(1)
 	if err != nil {
-		log.Fatalf("There was no systemd process found: %v", err)
+		return fmt.Errorf("not running on Unix: %v", err)
 	}
 
-	err = process.Signal(syscall.Signal(c.signal))
+	err = process.Signal(syscall.Signal(c.Signal))
+	// Either PID does not exist, or the signal does not work. Hoping for
+	// a decent enough error.
 	if err != nil {
-		log.Fatalf("Signal of SIGRTMIN+5 failed: %v", err)
+		return fmt.Errorf("signal of SIGRTMIN+5 failed: %v", err)
 	}
+	return nil
+}
+
+// NewSignalRebooter is the constructor which sets the signal number.
+// The constructor does not yet validate any input. It should be done in a later commit.
+func NewSignalRebooter(sig int) (*SignalRebooter, error) {
+	if sig < 1 {
+		return nil, fmt.Errorf("invalid signal: %v", sig)
+	}
+	return &SignalRebooter{Signal: sig}, nil
 }

pkg/timewindow/days.go

@@ -81,11 +81,11 @@ func parseWeekday(day string) (time.Weekday, error) {
 		if n >= 0 && n < 7 {
 			return time.Weekday(n), nil
 		}
-		return time.Sunday, fmt.Errorf("Invalid weekday, number out of range: %s", day)
+		return time.Sunday, fmt.Errorf("invalid weekday, number out of range: %s", day)
 	}
 
 	if weekday, ok := dayStrings[strings.ToLower(day)]; ok {
 		return weekday, nil
 	}
-	return time.Sunday, fmt.Errorf("Invalid weekday: %s", day)
+	return time.Sunday, fmt.Errorf("invalid weekday: %s", day)
 }

pkg/timewindow/timewindow.go

@@ -77,5 +77,5 @@ func parseTime(s string, loc *time.Location) (time.Time, error) {
 		}
 	}
 
-	return time.Now(), fmt.Errorf("Invalid time format: %s", s)
+	return time.Now(), fmt.Errorf("invalid time format: %s", s)
 }

pkg/util/util.go

@@ -1,23 +0,0 @@
-package util
-
-import (
-	"os/exec"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// NewCommand creates a new Command with stdout/stderr wired to our standard logger
-func NewCommand(name string, arg ...string) *exec.Cmd {
-	cmd := exec.Command(name, arg...)
-	cmd.Stdout = log.NewEntry(log.StandardLogger()).
-		WithField("cmd", cmd.Args[0]).
-		WithField("std", "out").
-		WriterLevel(log.InfoLevel)
-
-	cmd.Stderr = log.NewEntry(log.StandardLogger()).
-		WithField("cmd", cmd.Args[0]).
-		WithField("std", "err").
-		WriterLevel(log.WarnLevel)
-
-	return cmd
-}

tests/kind/create-reboot-sentinels.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-# USE KUBECTL_CMD to pass context and/or namespaces.
-KUBECTL_CMD="${KUBECTL_CMD:-kubectl}"
-SENTINEL_FILE="${SENTINEL_FILE:-/var/run/reboot-required}"
-
-echo "Creating reboot sentinel on all nodes"
-
-for nodename in $("$KUBECTL_CMD" get nodes -o name); do
-    docker exec "${nodename/node\//}" hostname
-    docker exec "${nodename/node\//}" touch "${SENTINEL_FILE}"
-done

tests/kind/main_test.go

@@ -0,0 +1,429 @@
+package kind
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"os/exec"
+	"strconv"
+	"testing"
+	"time"
+)
+
+const (
+	kuredDevImage string = "kured:dev"
+)
+
+// KindTest cluster deployed by each TestMain function, prepared to run a given test scenario.
+type KindTest struct {
+	kindConfigPath  string
+	clusterName     string
+	timeout         time.Duration
+	deployManifests []string
+	localImages     []string
+	logsDir         string
+	logBuffer       bytes.Buffer
+	testInstance    *testing.T // Maybe move this to testing.TB
+}
+
+func (k *KindTest) Write(p []byte) (n int, err error) {
+	k.testInstance.Helper()
+	k.logBuffer.Write(p)
+	return len(p), nil
+}
+
+func (k *KindTest) FlushLog() {
+	k.testInstance.Helper()
+	k.testInstance.Log(k.logBuffer.String())
+	k.logBuffer.Reset()
+}
+
+func (k *KindTest) RunCmd(cmdDetails ...string) error {
+	cmd := exec.Command(cmdDetails[0], cmdDetails[1:]...)
+	// by making KindTest a Writer, we can simply wire k to logs
+	// writing to k will write to proper logs.
+	cmd.Stdout = k
+	cmd.Stderr = k
+
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Option that can be passed to the NewKind function in order to change the configuration
+// of the test cluster
+type Option func(k *KindTest)
+
+// Deploy can be passed to NewKind to deploy extra components, in addition to the base deployment.
+func Deploy(manifest string) Option {
+	return func(k *KindTest) {
+		k.deployManifests = append(k.deployManifests, manifest)
+	}
+}
+
+// ExportLogs can be passed to NewKind to specify the folder where the kubernetes logs will be exported after the tests.
+func ExportLogs(folder string) Option {
+	return func(k *KindTest) {
+		k.logsDir = folder
+	}
+}
+
+// Timeout for long-running operations (e.g. deployments, readiness probes...)
+func Timeout(t time.Duration) Option {
+	return func(k *KindTest) {
+		k.timeout = t
+	}
+}
+
+// LocalImage is passed to NewKind to allow loading a local Docker image into the cluster
+func LocalImage(nameTag string) Option {
+	return func(k *KindTest) {
+		k.localImages = append(k.localImages, nameTag)
+	}
+}
+
+// NewKind creates a kind cluster given a name and set of Option instances.
+func NewKindTester(kindClusterName string, filePath string, t *testing.T, options ...Option) *KindTest {
+
+	k := &KindTest{
+		clusterName:    kindClusterName,
+		timeout:        10 * time.Minute,
+		kindConfigPath: filePath,
+		testInstance:   t,
+	}
+	for _, option := range options {
+		option(k)
+	}
+	return k
+}
+
+// Prepare the kind cluster.
+func (k *KindTest) Create() error {
+	err := k.RunCmd("kind", "create", "cluster", "--name", k.clusterName, "--config", k.kindConfigPath)
+
+	if err != nil {
+		return fmt.Errorf("failed to create cluster: %v", err)
+	}
+
+	for _, img := range k.localImages {
+		if err := k.RunCmd("kind", "load", "docker-image", "--name", k.clusterName, img); err != nil {
+			return fmt.Errorf("failed to load image: %v", err)
+		}
+	}
+	for _, mf := range k.deployManifests {
+		kubectlContext := fmt.Sprintf("kind-%v", k.clusterName)
+		if err := k.RunCmd("kubectl", "--context", kubectlContext, "apply", "-f", mf); err != nil {
+			return fmt.Errorf("failed to deploy manifest: %v", err)
+		}
+	}
+	return nil
+}
+
+func (k *KindTest) Destroy() error {
+	if k.logsDir != "" {
+		if err := k.RunCmd("kind", "export", "logs", k.logsDir, "--name", k.clusterName); err != nil {
+			return fmt.Errorf("failed to export logs: %v. will not teardown", err)
+		}
+	}
+
+	if err := k.RunCmd("kind", "delete", "cluster", "--name", k.clusterName); err != nil {
+		return fmt.Errorf("failed to destroy cluster: %v", err)
+	}
+	return nil
+}
+
+func TestE2EWithCommand(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"previous",
+		"current",
+		"next",
+	}
+	// Iterate over each Kubernetes version
+	for _, version := range kindClusterConfigs {
+		version := version
+		// Define a subtest for each combination
+		t.Run(version, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-command-%v-%v", version, randomInt)
+			kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds.yaml"))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
+				t.Fatalf("failed to create sentinels: %v", err)
+			}
+
+			if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
+				t.Fatalf("failed to follow reboot: %v", err)
+			}
+		})
+	}
+}
+
+func TestE2EWithSignal(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"previous",
+		"current",
+		"next",
+	}
+	// Iterate over each Kubernetes version
+	for _, version := range kindClusterConfigs {
+		version := version
+		// Define a subtest for each combination
+		t.Run(version, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-signal-%v-%v", version, randomInt)
+			kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds-signal.yaml"))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
+				t.Fatalf("failed to create sentinels: %v", err)
+			}
+
+			if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
+				t.Fatalf("failed to follow reboot: %v", err)
+			}
+		})
+	}
+}
+
+func TestE2EConcurrentWithCommand(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"previous",
+		"current",
+		"next",
+	}
+	// Iterate over each Kubernetes version
+	for _, version := range kindClusterConfigs {
+		version := version
+		// Define a subtest for each combination
+		t.Run(version, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-concurrentcommand-%v-%v", version, randomInt)
+			kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds-concurrent-command.yaml"))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
+				t.Fatalf("failed to create sentinels: %v", err)
+			}
+
+			if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
+				t.Fatalf("failed to follow reboot: %v", err)
+			}
+		})
+	}
+}
+
+func TestE2EConcurrentWithSignal(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"previous",
+		"current",
+		"next",
+	}
+	// Iterate over each Kubernetes version
+	for _, version := range kindClusterConfigs {
+		version := version
+		// Define a subtest for each combination
+		t.Run(version, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-concurrentsignal-%v-%v", version, randomInt)
+			kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds-concurrent-signal.yaml"))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
+				t.Fatalf("failed to create sentinels: %v", err)
+			}
+
+			if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
+				t.Fatalf("failed to follow reboot: %v", err)
+			}
+		})
+	}
+}
+
+func TestCordonningIsKept(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"concurrency1",
+		"concurrency2",
+	}
+	// Iterate over each test variant
+	for _, variant := range kindClusterConfigs {
+		variant := variant
+		// Define a subtest for each combination
+		t.Run(variant, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-cordon-%v-%v", variant, randomInt)
+			kindClusterConfigFile := "../../.github/kind-cluster-next.yaml"
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			var manifest string
+			if variant == "concurrency1" {
+				manifest = "testfiles/kured-ds-signal.yaml"
+			} else {
+				manifest = "testfiles/kured-ds-concurrent-signal.yaml"
+			}
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy(manifest))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", "testfiles/node-stays-as-cordonned.sh", kindContext); err != nil {
+				t.Fatalf("node did not reboot in time: %v", err)
+			}
+		})
+	}
+}
+func TestE2EBlocker(t *testing.T) {
+	t.Parallel()
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	var kindClusterConfigs = []string{
+		"podblocker",
+	}
+	// Iterate over each variant of the test
+	for _, variant := range kindClusterConfigs {
+		variant := variant
+		// Define a subtest for each combination
+		t.Run(variant, func(t *testing.T) {
+			t.Parallel() // Allow tests to run in parallel
+
+			randomInt := strconv.Itoa(rand.Intn(100))
+			kindClusterName := fmt.Sprintf("kured-e2e-cordon-%v-%v", variant, randomInt)
+			kindClusterConfigFile := "../../.github/kind-cluster-next.yaml"
+			kindContext := fmt.Sprintf("kind-%v", kindClusterName)
+
+			k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy(fmt.Sprintf("testfiles/kured-ds-%v.yaml", variant)))
+			defer k.FlushLog()
+
+			err := k.Create()
+			if err != nil {
+				t.Fatalf("Error creating cluster %v", err)
+			}
+			defer func(k *KindTest) {
+				err := k.Destroy()
+				if err != nil {
+					t.Fatalf("Error destroying cluster %v", err)
+				}
+			}(k)
+
+			k.Write([]byte("Now running e2e tests"))
+
+			if err := k.RunCmd("bash", fmt.Sprintf("testfiles/%v.sh", variant), kindContext); err != nil {
+				t.Fatalf("node blocker test did not succeed: %v", err)
+			}
+		})
+	}
+}

tests/kind/testfiles/create-reboot-sentinels.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+kubectl_flags=( )
+[[ "$1" != "" ]] && kubectl_flags=("${kubectl_flags[@]}" --context "$1")
+
+# To speed up the system, let's not kill the control plane.
+for nodename in $(${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes -o name | grep -v control-plane); do
+    echo "Creating reboot sentinel on $nodename"
+    docker exec "${nodename/node\//}" hostname
+    docker exec "${nodename/node\//}" touch "${SENTINEL_FILE:-/var/run/reboot-required}"
+done

tests/kind/testfiles/follow-coordinated-reboot.sh

@@ -1,11 +1,14 @@
 #!/usr/bin/env bash
 
-NODECOUNT=${NODECOUNT:-5}
-KUBECTL_CMD="${KUBECTL_CMD:-kubectl}"
+REBOOTCOUNT=${REBOOTCOUNT:-2} # By default we only create two sentinels in create-reboot-sentinels.
 DEBUG="${DEBUG:-false}"
 CONTAINER_NAME_FORMAT=${CONTAINER_NAME_FORMAT:-"chart-testing-*"}
 
+kubectl_flags=( )
+[[ "$1" != "" ]] && kubectl_flags=("${kubectl_flags[@]}" --context "$1")
+
 tmp_dir=$(mktemp -d -t kured-XXXX)
+
 function gather_logs_and_cleanup {
     if [[ -f "$tmp_dir"/node_output ]]; then
         rm "$tmp_dir"/node_output
@@ -18,15 +21,15 @@ function gather_logs_and_cleanup {
         # This is useful to see if containers have crashed.
         echo "docker ps -a:"
         docker ps -a
-	echo "docker journal logs"
-	journalctl -u docker --no-pager
+	      echo "docker journal logs"
+	      journalctl -u docker --no-pager
 
         # This is useful to see if the nodes have _properly_ rebooted.
         # It should show the reboot/two container starts per node.
-        for name in $(docker ps -a -f "name=${CONTAINER_NAME_FORMAT}" -q); do
+        for id in $(docker ps -a -q); do
             echo "############################################################"
-            echo "docker logs for container $name:"
-            docker logs "$name"
+            echo "docker logs for container $id:"
+            docker logs "$id"
         done
 
     fi
@@ -35,30 +38,28 @@ trap gather_logs_and_cleanup EXIT
 
 declare -A was_unschedulable
 declare -A has_recovered
-max_attempts="60"
-sleep_time=60
+max_attempts="200"
+sleep_time=5
 attempt_num=1
 
+# Get docker info of each of those kind containers. If one has crashed, restart it.
+
 set +o errexit
-echo "There are $NODECOUNT nodes in the cluster"
-until [ ${#was_unschedulable[@]} == "$NODECOUNT" ] && [ ${#has_recovered[@]} == "$NODECOUNT" ]
+echo "There are $REBOOTCOUNT nodes total needing reboot in the cluster"
+until [ ${#was_unschedulable[@]} == "$REBOOTCOUNT" ] && [ ${#has_recovered[@]} == "$REBOOTCOUNT" ]
 do
     echo "${#was_unschedulable[@]} nodes were removed from pool once:" "${!was_unschedulable[@]}"
     echo "${#has_recovered[@]} nodes removed from the pool are now back:" "${!has_recovered[@]}"
 
-    #"$KUBECTL_CMD" logs -n kube-system -l name=kured --ignore-errors > "$tmp_dir"/node_output
-    #if [[ "$DEBUG" == "true" ]]; then
-    #    echo "Kured pod logs:"
-    #    cat "$tmp_dir"/node_output
-    #fi
 
-    "$KUBECTL_CMD" get nodes -o custom-columns=NAME:.metadata.name,SCHEDULABLE:.spec.unschedulable --no-headers > "$tmp_dir"/node_output
+    ${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes -o custom-columns=NAME:.metadata.name,SCHEDULABLE:.spec.unschedulable --no-headers | grep -v control-plane > "$tmp_dir"/node_output
     if [[ "$DEBUG" == "true" ]]; then
         # This is useful to see if a node gets stuck after drain, and doesn't
         # come back up.
-        echo "Result of command $KUBECTL_CMD get nodes ... showing unschedulable nodes:"
+        echo "Result of command kubectl unschedulable nodes:"
         cat "$tmp_dir"/node_output
     fi
+
     while read -r node; do
         unschedulable=$(echo "$node" | grep true | cut -f 1 -d ' ')
         if [ -n "$unschedulable" ] && [ -z ${was_unschedulable["$unschedulable"]+x} ] ; then
@@ -70,9 +71,15 @@ do
             echo "$schedulable has recovered!"
             has_recovered["$schedulable"]=1
         fi
+
+        # If the container has crashed, restart it.
+        node_name=$(echo "$node" | cut -f 1 -d ' ')
+        stopped_container_id=$(docker container ls --filter=name="$node_name" --filter=status=exited -q)
+        if [ -n "$stopped_container_id" ]; then echo "Node $stopped_container_id needs restart"; docker start "$stopped_container_id"; echo "Container started."; fi
+
     done < "$tmp_dir"/node_output
 
-    if [[ "${#has_recovered[@]}" == "$NODECOUNT" ]]; then
+    if [[ "${#has_recovered[@]}" == "$REBOOTCOUNT" ]]; then
         echo "All nodes recovered."
         break
     else

tests/kind/testfiles/node-stays-as-cordonned.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+kubectl_flags=( )
+[[ "$1" != "" ]] && kubectl_flags=("${kubectl_flags[@]}" --context "$1")
+
+cordon() {
+  kubectl "${kubectl_flags[@]}" cordon "${precordonned_node}"
+}
+
+create_sentinel() {
+  docker exec "${precordonned_node}" touch "${SENTINEL_FILE:-/var/run/reboot-required}"
+  docker exec "${notcordonned_node}" touch "${SENTINEL_FILE:-/var/run/reboot-required}"
+}
+
+check_reboot_required() {
+  while true;
+  do
+    docker exec "${precordonned_node}" stat /var/run/reboot-required > /dev/null && echo "Reboot still required" || return 0
+    sleep 3
+  done
+}
+
+check_node_back_online_as_cordonned() {
+  sleep 5 # For safety, wait for 5 seconds, so that the kubectl command succeeds.
+  # This test might be giving us false positive until we work on reliability of the
+  # test.
+  while true;
+  do
+    result=$(kubectl "${kubectl_flags[@]}" get node "${precordonned_node}" --no-headers | awk '{print $2;}')
+    test "${result}" != "Ready,SchedulingDisabled" && echo "Node ${precordonned_node} in state ${result}" || return 0
+    sleep 3
+  done
+}
+
+check_node_back_online_as_uncordonned() {
+  while true;
+  do
+    result=$(kubectl "${kubectl_flags[@]}" get node "${notcordonned_node}" --no-headers | awk '{print $2;}')
+    test "${result}" != "Ready" && echo "Node ${notcordonned_node} in state ${result}" || return 0
+    sleep 3
+  done
+}
+### Start main
+
+worker_nodes=$(${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes -o custom-columns=name:metadata.name --no-headers | grep worker)
+precordonned_node=$(echo "$worker_nodes" | head -n 1)
+notcordonned_node=$(echo "$worker_nodes" | tail -n 1)
+
+# Wait for kured to install correctly
+sleep 15
+cordon
+create_sentinel
+check_reboot_required
+echo "Node has rebooted, but may take time to come back ready"
+check_node_back_online_as_cordonned
+check_node_back_online_as_uncordonned
+echo "Showing final node state"
+${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes
+echo "Test successful"

tests/kind/testfiles/podblocker.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+kubectl_flags=( )
+[[ "$1" != "" ]] && kubectl_flags=("${kubectl_flags[@]}" --context "$1")
+
+function gather_logs_and_cleanup {
+      for id in $(docker ps -q); do
+          echo "############################################################"
+          echo "docker logs for container $id:"
+          docker logs "$id"
+      done
+      ${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" logs ds/kured --all-pods -n kube-system
+}
+trap gather_logs_and_cleanup EXIT
+
+set +o errexit
+worker=$(${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" get nodes -o custom-columns=name:metadata.name --no-headers | grep worker | head -n 1)
+
+${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" label nodes "$worker" blocked-host=yes
+
+${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" apply -f - << EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+  labels:
+    app: blocker
+spec:
+  containers:
+    - name: nginx
+      image: nginx
+      imagePullPolicy: IfNotPresent
+  nodeSelector:
+    blocked-host: "yes"
+EOF
+
+docker exec "$worker" touch "${SENTINEL_FILE:-/var/run/reboot-required}"
+
+set -o errexit
+max_attempts="100"
+attempt_num=1
+sleep_time=5
+
+until ${KUBECTL_CMD:-kubectl} "${kubectl_flags[@]}" logs ds/kured --all-pods -n kube-system  | grep -i -e "Reboot.*blocked"
+do
+  if (( attempt_num == max_attempts )); then
+      echo "Attempt $attempt_num failed and there are no more attempts left!"
+      exit 1
+  else
+      echo "Did not find 'reboot blocked' in the log, retrying in $sleep_time seconds (Attempt #$attempt_num)"
+      sleep "$sleep_time"
+  fi
+  (( attempt_num++ ))
+done