Merge pull request #354 from evrardjp/test-prom

Add prometheus export metrics functional testing

.github/workflows/on-pr.yaml

@@ -211,3 +211,84 @@ jobs:
           retry_wait_seconds: 60
           # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE should all be = to cluster_size
           command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'"
+
+  test-prom:
+    name: "Test prometheus with latest code from HEAD"
+    runs-on: ubuntu-latest
+    # only build with oldest and newest supported, it should be good enough.
+    strategy:
+      fail-fast: false
+      matrix:
+        kubernetes:
+          - "1.20"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Find go version
+        run: |
+          GO_VERSION=$(awk '/^go/ {print $2};' go.mod)
+          echo "::set-output name=version::${GO_VERSION}"
+        id: awk_gomod
+      - name: Ensure go version
+        uses: actions/setup-go@v2
+        with:
+          go-version: "${{ steps.awk_gomod.outputs.version }}"
+      - name: Build artifacts
+        run: |
+          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image
+          make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart
+
+      - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions"
+        run: |
+          sudo bash << EOF
+              cp /etc/docker/daemon.json /etc/docker/daemon.json.old
+              echo '{}' > /etc/docker/daemon.json
+              systemctl restart docker || journalctl --no-pager -n 500
+              systemctl status docker
+          EOF
+
+      # Default name for helm/kind-action kind clusters is "chart-testing"
+      - name: Create 1 node kind cluster
+        uses: helm/kind-action@master
+
+      - name: Preload previously built images onto kind cluster
+        run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing
+
+      - name: Deploy kured on default namespace with its helm chart
+        run: |
+          # Documented in official helm doc to live on the edge
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          # Refresh bins
+          hash -r
+          helm install kured ./charts/kured/ --wait --values ./charts/kured/ci/prometheus-values.yaml
+          kubectl config set-context kind-chart-testing
+          kubectl get ds --all-namespaces
+          kubectl describe ds kured
+
+      - name: Ensure kured is ready
+        uses: nick-invision/retry@v2.4.0
+        with:
+          timeout_minutes: 10
+          max_attempts: 10
+          retry_wait_seconds: 60
+          # DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE
+          command: "kubectl get ds kured | grep -E 'kured.*1.*1.*1.*1.*1' "
+
+      - name: Get metrics (healthy)
+        uses: nick-invision/retry@v2.4.0
+        with:
+          timeout_minutes: 2
+          max_attempts: 12
+          retry_wait_seconds: 5
+          command: "./tests/kind/test-metrics.sh 0"
+
+      - name: Create reboot sentinel files
+        run: |
+          ./tests/kind/create-reboot-sentinels.sh
+
+      - name: Get metrics (need reboot)
+        uses: nick-invision/retry@v2.4.0
+        with:
+          timeout_minutes: 15        
+          max_attempts: 10
+          retry_wait_seconds: 60
+          command: "./tests/kind/test-metrics.sh 1"

charts/kured/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 appVersion: "1.6.1"
 description: A Helm chart for kured
 name: kured
-version: 2.4.2
+version: 2.4.3
 home: https://github.com/weaveworks/kured
 maintainers:
   - name: ckotzbauer

charts/kured/ci/prometheus-values.yaml

@@ -0,0 +1,13 @@
+# This is tested twice:
+# Basic install test with chart-testing (on charts PRs)
+# Functional testing in PRs (other PRs)
+
+service:
+  create: true
+  name: kured-prometheus-endpoint
+  port: 8080
+  type: NodePort
+  nodePort: 30000
+
+# Do not override the configuration: period in this, so that
+# We can test prometheus exposed metrics without rebooting.

charts/kured/templates/service.yaml

@@ -16,11 +16,14 @@ metadata:
   {{- end }}
   {{- end }}
 spec:
-  type: ClusterIP
+  type: {{ .Values.service.type }}
   ports:
     - name: metrics
       port: {{ .Values.service.port }}
       targetPort: 8080
+      {{- if eq .Values.service.type "NodePort" }}
+      nodePort: {{ .Values.service.nodePort }}
+      {{- end }}
   selector:
     {{- include "kured.matchLabels" . | nindent 4 }}
 {{- end }}

charts/kured/values.yaml

@@ -64,6 +64,7 @@ service:
   port: 8080
   annotations: {}
   name: ""
+  type: ClusterIP
 
 podLabels: {}
 

tests/kind/test-metrics.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+expected="$1"
+if [[ "$expected" != "0" && "$expected" != "1" ]]; then
+    echo "You should give an argument to this script, the gauge value (0 or 1)"
+    exit 1
+fi
+
+HOST="${HOST:-localhost}"
+PORT="${PORT:-30000}"
+NODENAME="${NODENAME-chart-testing-control-plane}"
+
+reboot_required=$(docker exec "$NODENAME" curl "http://$HOST:$PORT/metrics" | awk '/^kured_reboot_required/{print $2}')
+if [[ "$reboot_required" == "$expected" ]]; then
+    echo "Test success"
+else
+    echo "Test failed"
+    exit 1
+fi