[stable/minio] fix serviceMonitor authentication (#20896)

Signed-off-by: Bruno Clermont <bruno@robotinfra.com>

stable/minio/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 description: MinIO is a high performance data infrastructure for machine learning, analytics and application data workloads.
 name: minio
-version: 5.0.18
+version: 5.0.19
 appVersion: master
 keywords:
 - storage

stable/minio/templates/post-install-prometheus-metrics-job.yaml

@@ -0,0 +1,110 @@
+{{- if .Values.metrics.serviceMonitor.enabled }}
+{{- $fullName := include "minio.fullname" . -}}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ $fullName }}-update-prometheus-secret
+  labels:
+    app: {{ template "minio.name" . }}-update-prometheus-secret
+    chart: {{ template "minio.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded
+  {{ toYaml .Values.updatePrometheusJob.annotations | indent 4 }}
+spec:
+  template:
+    metadata:
+      labels:
+        app: {{ template "minio.name" . }}-update-prometheus-secret
+        release: {{ .Release.Name }}
+{{- if .Values.podLabels }}
+{{ toYaml .Values.podLabels | indent 8 }}
+{{- end }}
+    spec:
+{{- if .Values.serviceAccount.create }}
+      serviceAccountName: {{ $fullName }}-update-prometheus-secret
+{{- end }}
+      restartPolicy: OnFailure
+{{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+{{- end }}
+{{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+{{- end }}
+{{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+{{- end }}
+      volumes:
+        - name: workdir
+          emptyDir: {}
+      initContainers:
+        - name: minio-mc
+          image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}"
+          imagePullPolicy: {{ .Values.mcImage.pullPolicy }}
+          command:
+            - /bin/sh
+            - "-c"
+            - mc admin prometheus generate target --json --no-color -q > /workdir/mc.json
+          env:
+            # mc admin prometheus generate don't really connect to remote server, TLS cert isn't required
+            - name: MC_HOST_target
+              value: http{{ if .Values.tls.enabled }}s{{ end }}://{{ .Values.accessKey }}:{{ .Values.secretKey }}@{{ $fullName }}:{{ .Values.service.port }}
+          volumeMounts:
+            - name: workdir
+              mountPath: /workdir
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+        # extract bearerToken from mc admin output
+        - name: jq
+          image: "{{ .Values.helmKubectlJqImage.repository }}:{{ .Values.helmKubectlJqImage.tag }}"
+          imagePullPolicy: {{ .Values.helmKubectlJqImage.pullPolicy }}
+          command:
+            - /bin/sh
+            - "-c"
+            - jq -e -c -j -r .bearerToken < /workdir/mc.json > /workdir/token
+          volumeMounts:
+            - name: workdir
+              mountPath: /workdir
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+        - name: kubectl-create
+          image: "{{ .Values.helmKubectlJqImage.repository }}:{{ .Values.helmKubectlJqImage.tag }}"
+          imagePullPolicy: {{ .Values.helmKubectlJqImage.pullPolicy }}
+          command:
+            - /bin/sh
+            - "-c"
+            # The following script does:
+            # - get the servicemonitor that need this secret and copy some metadata and create the ownerreference for the secret file
+            # - create the secret
+            # - merge both json
+            - >
+              kubectl -n {{ .Release.Namespace }} get servicemonitor {{ $fullName }} -o json |
+                jq -c '{metadata: {name: "{{ $fullName }}-prometheus", namespace: .metadata.namespace, labels: {app: .metadata.labels.app, release: .metadata.labels.release}, ownerReferences: [{apiVersion: .apiVersion, kind: .kind, blockOwnerDeletion: true, controller: true, uid: .metadata.uid, name: .metadata.name}]}}' > /workdir/metadata.json &&
+              kubectl create secret generic {{ $fullName }}-prometheus --from-file=token=/workdir/token --dry-run -o json > /workdir/secret.json &&
+              cat /workdir/secret.json /workdir/metadata.json | jq -s add > /workdir/object.json
+          volumeMounts:
+            - name: workdir
+              mountPath: /workdir
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+      containers:
+        - name: kubectl-apply
+          image: "{{ .Values.helmKubectlJqImage.repository }}:{{ .Values.helmKubectlJqImage.tag }}"
+          imagePullPolicy: {{ .Values.helmKubectlJqImage.pullPolicy }}
+          command:
+            - kubectl
+            - apply
+            - "-f"
+            - /workdir/object.json
+          volumeMounts:
+            - name: workdir
+              mountPath: /workdir
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+{{- end }}

stable/minio/templates/post-install-prometheus-metrics-role.yaml

@@ -0,0 +1,38 @@
+{{- if .Values.serviceAccount.create -}}
+{{- $fullName := include "minio.fullname" . -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $fullName }}-update-prometheus-secret
+  labels:
+    app: {{ template "minio.name" . }}-update-prometheus-secret
+    chart: {{ template "minio.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+      - update
+      - patch
+    resourceNames:
+      - {{ $fullName }}-prometheus
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - servicemonitors
+    verbs:
+      - get
+    resourceNames:
+      - {{ $fullName }}
+{{- end -}}

stable/minio/templates/post-install-prometheus-metrics-rolebinding.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.serviceAccount.create -}}
+{{- $fullName := include "minio.fullname" . -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $fullName }}-update-prometheus-secret
+  labels:
+    app: {{ template "minio.name" . }}-update-prometheus-secret
+    chart: {{ template "minio.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $fullName }}-update-prometheus-secret
+subjects:
+  - kind: ServiceAccount
+    name: {{ $fullName }}-update-prometheus-secret
+    namespace: {{ .Release.Namespace }}
+{{- end -}}

stable/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create -}}
+{{- $fullName := include "minio.fullname" . -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ $fullName }}-update-prometheus-secret
+  labels:
+    app: {{ template "minio.name" . }}-update-prometheus-secret
+    chart: {{ template "minio.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+{{- end -}}

stable/minio/templates/servicemonitor.yaml

@@ -28,6 +28,9 @@ spec:
       {{- if .Values.metrics.serviceMonitor.scrapeTimeout }}
       scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }}
       {{- end }}
+      bearerTokenSecret:
+        name: {{ template "minio.fullname" . }}-prometheus
+        key: token
   namespaceSelector:
     matchNames:
       - {{ .Release.Namespace }}

stable/minio/values.yaml

@@ -25,6 +25,14 @@ mcImage:
   tag: RELEASE.2020-03-14T01-23-37Z
   pullPolicy: IfNotPresent
 
+## Set default image, imageTag, and imagePullPolicy for the `jq` (the JSON
+## process used to create secret for prometheus ServiceMonitor).
+##
+helmKubectlJqImage:
+  repository: bskim45/helm-kubectl-jq
+  tag: 3.1.0
+  pullPolicy: IfNotPresent
+
 ## minio server mode, i.e. standalone or distributed.
 ## Distributed Minio ref: https://docs.minio.io/docs/distributed-minio-quickstart-guide
 ##
@@ -226,6 +234,10 @@ buckets: []
 makeBucketJob:
   annotations:
 
+## Additional Annotations for the Kubernetes Batch (update-prometheus-secret)
+updatePrometheusJob:
+  annotations:
+
 s3gateway:
   enabled: false
   replicas: 4