diff --git a/stable/rabbitmq-ha/Chart.yaml b/stable/rabbitmq-ha/Chart.yaml index 09dd5817e2..0dfea9f2ce 100644 --- a/stable/rabbitmq-ha/Chart.yaml +++ b/stable/rabbitmq-ha/Chart.yaml @@ -1,7 +1,7 @@ name: rabbitmq-ha apiVersion: v1 appVersion: 3.7.4 -version: 1.5.2 +version: 1.6.0 description: Highly available RabbitMQ cluster, the open source message broker software that implements the Advanced Message Queuing Protocol (AMQP). keywords: diff --git a/stable/rabbitmq-ha/README.md b/stable/rabbitmq-ha/README.md index 0c1fc3e128..c3607b4779 100644 --- a/stable/rabbitmq-ha/README.md +++ b/stable/rabbitmq-ha/README.md @@ -66,7 +66,7 @@ and their default values. | Parameter | Description | Default | |------------------------------------|-----------------------------------------------------------------|----------------------------------------------------------| | `customConfigMap` | Use a custom ConfigMap | `false` | -| `customSecret` | Use a custom Secret | `false` | +| `existingSecret`     | Use an existing secret for password & erlang cookie               | `""`                               | | `image.pullPolicy` | Image pull policy | `Always` if `image` tag is `latest`, else `IfNotPresent` | | `image.repository` | RabbitMQ container image repository | `rabbitmq` | | `image.tag` | RabbitMQ container image tag | `3.7-alpine` | @@ -78,10 +78,24 @@ and their default values. | `persistentVolume.size` | Persistent volume size | `8Gi` | | `persistentVolume.storageClass` | Persistent volume storage class | `-` | | `podAntiAffinity` | Pod antiaffinity, `hard` or `soft` | `hard` | +| `prometheus.exporter.enabled` | Configures Prometheus Exporter to expose and scrape stats | `true` | +| `prometheus.exporter.env` | Environment variables to set for Exporter container | `{}` | +| `prometheus.exporter.image.repository` | Prometheus Exporter repository | `kbudde/rabbitmq-exporter` | +| `prometheus.exporter.image.tag` | Image Tag | `latest` | +| `prometheus.exporter.image.pullPolicy` | Image Pull Policy | `IfNotPresent` | +| `prometheus.exporter.port` | Port Prometheus scrapes for metrics | `9090` | +| `prometheus.exporter.resources` | Resource Limits for Prometheus Exporter | `{}` | +| `prometheus.operator.enabled` | Are you using Prometheus Operator? [Blog Post](https://coreos.com/blog/the-prometheus-operator.html) | `true` | +| `prometheus.operator.alerts.enabled` | Create default Alerts for RabbitMQ | `true` | +| `prometheus.operator.alerts.selector` | Selector to find ConfigMaps and create Prometheus Alerts | `alert-rules` | +| `prometheus.operator.alerts.labels` | Labels to add to Alerts | `{}` | +| `prometheus.operator.serviceMonitor.interval` | How often Prometheus Scrapes metrics | `10s` | +| `prometheus.operator.serviceMonitor.namespace` | Namespace which Prometheus is installed | `monitoring` | +| `prometheus.operator.serviceMonitor.selector` | Label Selector for Prometheus to find ServiceMonitors | `{ prometheus: kube-prometheus }` | | `rabbitmqCert.enabled` | Mount a Secret container certificates | `false` | -| `rabbitmqCert.existingSecret`     | Name of an existing `Secret` to mount for amqps               | ``                               | | `rabbitmqCert.cacertfile` | base64 encoded CA certificate (overwrites existing Secret) | `` | | `rabbitmqCert.certfile` | base64 encoded server certificate (overwrites existing Secret) | `` | +| `rabbitmqCert.existingSecret`     | Name of an existing `Secret` to mount for amqps               | `""`                               | | `rabbitmqCert.keyfile` | base64 encoded server private key (overwrites existing Secret) | `` | | `rabbitmqEpmdPort` | EPMD port used for cross cluster replication | `4369` | | `rabbitmqErlangCookie` | Erlang cookie | _random 32 character long alphanumeric string_ | @@ -101,10 +115,10 @@ and their default values. | `rabbitmqWebSTOMPPlugin.config` | STOMP over websocket configuration | `` | | `rabbitmqWebSTOMPPlugin.enabled` | Enable STOMP over websocket plugin | `false` | | `rbac.create` | If true, create & use RBAC resources | `true` | +| `replicaCount` | Number of replica | `3` | +| `resources` | CPU/Memory resource requests/limits | `{}` | `serviceAccount.create` | Create service account | `true` | | `serviceAccount.name` | Service account name to use | _name of the release_ | -| `replicaCount` | Number of replica | `3` | -| `resources` | CPU/Memory resource requests/limits | `{}` | | `service.annotations` | Annotations to add to the service | `{}` | | `service.clusterIP` | IP address to assign to the service | `""` | | `service.externalIPs` | Service external IP addresses | `[]` | @@ -182,4 +196,9 @@ $ helm install --name my-release --set customConfigMap=true stable/rabbitmq-ha ### Custom Secret -Similar to custom ConfigMap, `customSecret` can be used to override the default secret.yaml provided. +Similar to custom ConfigMap, `existingSecret` can be used to override the default secret.yaml provided, and +`rabbitmqCert.existingSecret` can be used to override the default certificates. + +### Prometheus Monitoring & Alerts + +Prometheus and its features can be enabled by setting `prometheus.enabled` to `true`. See values.yaml for more details and configuration options diff --git a/stable/rabbitmq-ha/templates/NOTES.txt b/stable/rabbitmq-ha/templates/NOTES.txt index bf4c71f919..92fbcacc76 100644 --- a/stable/rabbitmq-ha/templates/NOTES.txt +++ b/stable/rabbitmq-ha/templates/NOTES.txt @@ -3,8 +3,13 @@ Credentials: Username : {{ .Values.rabbitmqUsername }} + {{ if .Values.existingSecret }} + Password : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.secretName" . }} -o jsonpath="{.data.rabbitmq-password}" | base64 --decode) + ErLang Cookie : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.secretName" . }} -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 --decode) + {{ else }} Password : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.fullname" . }} -o jsonpath="{.data.rabbitmq-password}" | base64 --decode) ErLang Cookie : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.fullname" . }} -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 --decode) + {{ end }} RabbitMQ can be accessed within the cluster on port {{ .Values.rabbitmqNodePort }} at {{ template "rabbitmq-ha.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local diff --git a/stable/rabbitmq-ha/templates/_helpers.tpl b/stable/rabbitmq-ha/templates/_helpers.tpl index 0069a87aa3..161dcfad1c 100644 --- a/stable/rabbitmq-ha/templates/_helpers.tpl +++ b/stable/rabbitmq-ha/templates/_helpers.tpl @@ -34,3 +34,17 @@ Create the name of the service account to use {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +Generate chart secret name +*/}} +{{- define "rabbitmq-ha.secretName" -}} +{{ default (include "rabbitmq-ha.fullname" .) .Values.existingSecret }} +{{- end -}} + +{{/* +Generate chart ssl secret name +*/}} +{{- define "rabbitmq-ha.certSecretName" -}} +{{ default (include "rabbitmq-ha.fullname" .) .Values.rabbitmqCert.existingSecret }} +{{- end -}} diff --git a/stable/rabbitmq-ha/templates/alerts.yaml b/stable/rabbitmq-ha/templates/alerts.yaml new file mode 100644 index 0000000000..756dacf736 --- /dev/null +++ b/stable/rabbitmq-ha/templates/alerts.yaml @@ -0,0 +1,105 @@ +{{ if and .Values.prometheus.operator.alerts.enabled .Values.prometheus.exporter.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-rabbitmq-alerts + namespace: monitoring + labels: + app: "rabbitmq" + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.prometheus.operator.serviceMonitor.selector }} +{{ toYaml .Values.prometheus.operator.serviceMonitor.selector | indent 4 }} +{{- end }} +{{- if .Values.prometheus.operator.alerts.selector }} +{{ toYaml .Values.prometheus.operator.alerts.selector | indent 4 }} +{{- end }} +data: + rabbitmq.rules: |- + groups: + - name: rabbitmq-alerts + rules: + - alert: RabbitMqClusterNodeDown + expr: rabbitmq_up{service="{{ template "rabbitmq-ha.fullname" . }}"} == 0 + for: 5m + labels: + installed_by: {{ .Release.Name }} + severity: critical +{{- if .Values.prometheus.operator.alerts.labels }} +{{ toYaml .Values.prometheus.operator.alerts.labels | indent 10 }} +{{- end }} + annotations: + description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} is down + summary: RabbitMQ Node Is Down + - alert: RabbitMqClusterNotAllNodesRunning + expr: sum(rabbitmq_up{service="{{ template "rabbitmq-ha.fullname" . }}"}) by (service) < {{ .Values.replicaCount }} + for: 5m + labels: + installed_by: {{ .Release.Name }} + severity: critical + team: devops + annotations: + description: Some RabbitMQ Cluster Nodes Are Down in Service {{`{{ $labels.namespace }}`}}/{{`{{ $labels.service}}`}} + summary: Some RabbitMQ Cluster Nodes Are Down in Service {{`{{ $labels.namespace }}`}}/{{`{{ $labels.service}}`}} + - alert: RabbitMqDiskSpaceAlarm + expr: rabbitmq_node_disk_free_alarm{service="{{ template "rabbitmq-ha.fullname" . }}"} == 1 + for: 1m + labels: + installed_by: {{ .Release.Name }} + severity: critical +{{- if .Values.prometheus.operator.alerts.labels }} +{{ toYaml .Values.prometheus.operator.alerts.labels | indent 10 }} +{{- end }} + annotations: + description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} Disk Space Alarm is going off. Which means the node hit highwater mark and has cut off network connectivity, see RabbitMQ WebUI + summary: RabbitMQ is Out of Disk Space + - alert: RabbitMqMemoryAlarm + expr: rabbitmq_node_mem_alarm{service="{{ template "rabbitmq-ha.fullname" . }}"} == 1 + for: 1m + labels: + installed_by: {{ .Release.Name }} + severity: critical +{{- if .Values.prometheus.operator.alerts.labels }} +{{ toYaml .Values.prometheus.operator.alerts.labels | indent 10 }} +{{- end }} + annotations: + description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} High Memory Alarm is going off. Which means the node hit highwater mark and has cut off network connectivity, see RabbitMQ WebUI + summary: RabbitMQ is Out of Memory + - alert: RabbitMqMemoryUsageHigh + expr: (rabbitmq_node_mem_used{service="{{ template "rabbitmq-ha.fullname" . }}"} / rabbitmq_node_mem_limit{service="{{ template "rabbitmq-ha.fullname" . }}"}) > .9 + for: 1m + labels: + installed_by: {{ .Release.Name }} + severity: critical +{{- if .Values.prometheus.operator.alerts.labels }} +{{ toYaml .Values.prometheus.operator.alerts.labels | indent 10 }} +{{- end }} + annotations: + description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} Memory Usage > 90% + summary: RabbitMQ Node > 90% Memory Usage + - alert: RabbitMqFileDescriptorsLow + expr: (rabbitmq_fd_used{service="{{ template "rabbitmq-ha.fullname" . }}"} / rabbitmq_fd_total{service="{{ template "rabbitmq-ha.fullname" . }}"}) > .9 + for: 5m + labels: + installed_by: {{ .Release.Name }} + severity: critical +{{- if .Values.prometheus.operator.alerts.labels }} +{{ toYaml .Values.prometheus.operator.alerts.labels | indent 10 }} +{{- end }} + annotations: + description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} File Descriptor Usage > 90% + summary: RabbitMQ Low File Descriptor Available + - alert: RabbitMqDiskSpaceLow + expr: predict_linear(rabbitmq_node_disk_free{service="{{ template "rabbitmq-ha.fullname" . }}"}[15m], 1 * 60 * 60) < rabbitmq_node_disk_free_limit{service="{{ template "rabbitmq-ha.fullname" . }}"} + for: 5m + labels: + installed_by: {{ .Release.Name }} + severity: critical +{{- if .Values.prometheus.operator.alerts.labels }} +{{ toYaml .Values.prometheus.operator.alerts.labels | indent 10 }} +{{- end }} + annotations: + description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} will hit disk limit in the next hr based on last 15 mins trend. + summary: RabbitMQ is Low on Disk Space and will Run Out in the next hour +{{ end }} diff --git a/stable/rabbitmq-ha/templates/configmap.yaml b/stable/rabbitmq-ha/templates/configmap.yaml index ad17480701..bb7aad7350 100644 --- a/stable/rabbitmq-ha/templates/configmap.yaml +++ b/stable/rabbitmq-ha/templates/configmap.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.customConfigMap }} +{{- if not .Values.existingConfigMap }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/stable/rabbitmq-ha/templates/secret.yaml b/stable/rabbitmq-ha/templates/secret.yaml index 4604a09797..0d78ccef3e 100644 --- a/stable/rabbitmq-ha/templates/secret.yaml +++ b/stable/rabbitmq-ha/templates/secret.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.customSecret }} +{{ if not .Values.existingSecret }} apiVersion: v1 kind: Secret metadata: @@ -23,6 +23,7 @@ data: {{ else }} rabbitmq-erlang-cookie: {{ randAlphaNum 32 | b64enc | quote }} {{ end }} +{{ end }} {{- if and .Values.rabbitmqCert.enabled (not .Values.rabbitmqCert.existingSecret) }} --- apiVersion: v1 @@ -43,4 +44,3 @@ data: cert.pem: {{ .Values.rabbitmqCert.certfile | quote }} key.pem: {{ .Values.rabbitmqCert.keyfile | quote }} {{- end }} -{{- end }} diff --git a/stable/rabbitmq-ha/templates/service.yaml b/stable/rabbitmq-ha/templates/service.yaml index 241ccb822f..f243531a93 100644 --- a/stable/rabbitmq-ha/templates/service.yaml +++ b/stable/rabbitmq-ha/templates/service.yaml @@ -78,6 +78,12 @@ spec: port: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} targetPort: amqps {{- end }} + {{ if .Values.prometheus.exporter.enabled }} + - name: exporter + protocol: TCP + port: {{ .Values.prometheus.exporter.port }} + targetPort: exporter + {{ end }} selector: app: {{ template "rabbitmq-ha.name" . }} release: {{ .Release.Name }} diff --git a/stable/rabbitmq-ha/templates/servicemonitor.yaml b/stable/rabbitmq-ha/templates/servicemonitor.yaml new file mode 100644 index 0000000000..967e578cd1 --- /dev/null +++ b/stable/rabbitmq-ha/templates/servicemonitor.yaml @@ -0,0 +1,21 @@ +{{ if and .Values.prometheus.exporter.enabled .Values.prometheus.operator.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "rabbitmq-ha.fullname" . }} + namespace: {{ .Values.prometheus.operator.serviceMonitor.namespace }} + labels: +{{- if .Values.prometheus.operator.serviceMonitor.selector }} +{{ toYaml .Values.prometheus.operator.serviceMonitor.selector | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app: {{ template "rabbitmq-ha.name" . }} + release: {{ .Release.Name }} + endpoints: + - port: exporter + interval: {{ .Values.prometheus.operator.serviceMonitor.interval }} + namespaceSelector: + any: true +{{ end }} diff --git a/stable/rabbitmq-ha/templates/statefulset.yaml b/stable/rabbitmq-ha/templates/statefulset.yaml index 532a5a3489..0a803d5b3e 100644 --- a/stable/rabbitmq-ha/templates/statefulset.yaml +++ b/stable/rabbitmq-ha/templates/statefulset.yaml @@ -24,9 +24,13 @@ spec: {{ toYaml .Values.extraLabels | indent 8 }} {{- end }} annotations: - {{- if not .Values.customConfigMap }} + {{- if not .Values.existingConfigMap }} checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} {{- end }} +{{- if and .Values.prometheus.exporter.enabled (not .Values.prometheus.operator.enabled) }} + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.prometheus.exporter.port | quote }} +{{- end }} spec: terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} serviceAccountName: {{ template "rabbitmq-ha.serviceAccountName" . }} @@ -119,7 +123,7 @@ spec: - name: RABBITMQ_ERLANG_COOKIE valueFrom: secretKeyRef: - name: {{ template "rabbitmq-ha.fullname" . }} + name: {{ template "rabbitmq-ha.secretName" . }} key: rabbitmq-erlang-cookie {{- if .Values.rabbitmqHipeCompile }} - name: RABBITMQ_HIPE_COMPILE @@ -130,7 +134,7 @@ spec: - name: RABBITMQ_DEFAULT_PASS valueFrom: secretKeyRef: - name: {{ template "rabbitmq-ha.fullname" . }} + name: {{ template "rabbitmq-ha.secretName" . }} key: rabbitmq-password - name: RABBITMQ_DEFAULT_VHOST value: {{ .Values.rabbitmqVhost | quote }} @@ -145,6 +149,29 @@ spec: - name: cert mountPath: /etc/cert {{- end }} + {{ if .Values.prometheus.exporter.enabled }} + - name: {{ .Chart.Name }}-exporter + image: {{ .Values.prometheus.exporter.image.repository }}:{{ .Values.prometheus.exporter.image.tag }} + imagePullPolicy: {{ .Values.prometheus.exporter.image.pullPolicy }} + ports: + - name: exporter + protocol: TCP + containerPort: {{ .Values.prometheus.exporter.port }} + env: + - name: RABBIT_USER + value: {{ .Values.rabbitmqUsername | quote }} + - name: RABBIT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "rabbitmq-ha.secretName" . }} + key: rabbitmq-password + {{- range $key, $value := .Values.prometheus.exporter.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: +{{ toYaml .Values.prometheus.exporter.resources | indent 12 }} + {{ end }} {{- if .Values.nodeSelector }} nodeSelector: {{ toYaml .Values.nodeSelector | indent 8 }} @@ -184,7 +211,7 @@ spec: - name: cert secret: defaultMode: 420 - secretName: {{ .Values.rabbitmqCert.existingSecret | default (printf "%s-cert" (include "rabbitmq-ha.fullname" . )) }} + secretName: {{ template "rabbitmq-ha.certSecretName" . }} {{- end }} {{- if .Values.persistentVolume.enabled }} volumeClaimTemplates: diff --git a/stable/rabbitmq-ha/values.yaml b/stable/rabbitmq-ha/values.yaml index cd55ea1145..3f637c5f82 100644 --- a/stable/rabbitmq-ha/values.yaml +++ b/stable/rabbitmq-ha/values.yaml @@ -41,8 +41,8 @@ rabbitmqHipeCompile: false rabbitmqCert: enabled: false - ## Specify an existing secret to use - existingSecret: | + # Specifies an existing secret to be used for SSL Certs + existingSecret: "" ## Create a new secret using these values cacertfile: | @@ -214,16 +214,12 @@ podAntiAffinity: soft ## Create default configMap ## -customConfigMap: false +existingConfigMap: false ## Add additional labels to all resources ## extraLabels: {} -## Create default Secret -## -customSecret: false - ## Role Based Access ## Ref: https://kubernetes.io/docs/admin/authorization/rbac/ ## @@ -269,3 +265,57 @@ readinessProbe: initialDelaySeconds: 10 timeoutSeconds: 3 periodSeconds: 5 + +# Specifies an existing secret to be used for RMQ password and Erlang Cookie +existingSecret: "" + +prometheus: + ## Configures Prometheus Exporter to expose and scrape stats. + exporter: + enabled: false + env: {} + image: + repository: kbudde/rabbitmq-exporter + tag: v0.28.0 + pullPolicy: IfNotPresent + + ## Port Prometheus scrapes for metrics + port: 9090 + + ## Allow overriding of container resources + resources: {} + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # cpu: 100m + # memory: 100Mi + + ## Prometheus is using Operator. Setting to true will create Operator specific resources like ServiceMonitors and Alerts + operator: + ## Are you using Prometheus Operator? [Blog Post](https://coreos.com/blog/the-prometheus-operator.html) + enabled: true + + ## Configures Alerts, which will be setup via Prometheus Operator / ConfigMaps. + alerts: + ## Prometheus exporter must be enabled as well + enabled: true + + ## Selector must be configured to match Prometheus Install, defaulting to whats done by Prometheus Operator + ## See [CoreOS Prometheus Chart](https://github.com/coreos/prometheus-operator/tree/master/helm) + selector: + role: alert-rules + labels: {} + + serviceMonitor: + ## Interval at which Prometheus scrapes RabbitMQ Exporter + interval: 10s + + # Namespace Prometheus is installed in + namespace: monitoring + + ## Defaults to whats used if you follow CoreOS [Prometheus Install Instructions](https://github.com/coreos/prometheus-operator/tree/master/helm#tldr) + ## [Prometheus Selector Label](https://github.com/coreos/prometheus-operator/blob/master/helm/prometheus/templates/prometheus.yaml#L65) + ## [Kube Prometheus Selector Label](https://github.com/coreos/prometheus-operator/blob/master/helm/kube-prometheus/values.yaml#L298) + selector: + prometheus: kube-prometheus