From d9d1582ae8ce5ffb2a71009a78b8dc7a0d88ae4e Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Mon, 27 Aug 2018 22:29:19 -0500 Subject: [PATCH] Fix high CPU usage (#7378) Signed-off-by: Steven Sheehy --- stable/rabbitmq-ha/Chart.yaml | 2 +- stable/rabbitmq-ha/README.md | 7 ++- stable/rabbitmq-ha/templates/NOTES.txt | 9 +--- stable/rabbitmq-ha/templates/_helpers.tpl | 38 ++++++++++++++ stable/rabbitmq-ha/templates/alerts.yaml | 2 +- stable/rabbitmq-ha/templates/configmap.yaml | 6 ++- stable/rabbitmq-ha/templates/secret.yaml | 14 ++--- stable/rabbitmq-ha/templates/statefulset.yaml | 52 ++++++++++++------- stable/rabbitmq-ha/values.yaml | 28 ++++++++++ 9 files changed, 120 insertions(+), 38 deletions(-) diff --git a/stable/rabbitmq-ha/Chart.yaml b/stable/rabbitmq-ha/Chart.yaml index 25181bc362..42a625ab88 100644 --- a/stable/rabbitmq-ha/Chart.yaml +++ b/stable/rabbitmq-ha/Chart.yaml @@ -1,7 +1,7 @@ name: rabbitmq-ha apiVersion: v1 appVersion: 3.7.4 -version: 1.8.2 +version: 1.9.0 description: Highly available RabbitMQ cluster, the open source message broker software that implements the Advanced Message Queuing Protocol (AMQP). keywords: diff --git a/stable/rabbitmq-ha/README.md b/stable/rabbitmq-ha/README.md index aafc1a6751..8bc0853667 100644 --- a/stable/rabbitmq-ha/README.md +++ b/stable/rabbitmq-ha/README.md @@ -67,10 +67,13 @@ and their default values. |------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------| | `existingConfigMap` | Use an existing ConfigMap | `false` | | `existingSecret` | Use an existing secret for password & erlang cookie | `""`                               | +| `extraConfig` | Additional configuration to add to default configmap | `` | | `image.pullPolicy` | Image pull policy | `Always` if `image` tag is `latest`, else `IfNotPresent` | | `image.repository` | RabbitMQ container image repository | `rabbitmq` | | `image.tag` | RabbitMQ container image tag | `3.7-alpine` | | `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` | +| `managementPassword` | Management user password. Should be changed from default | `E9R3fjZm4ejFkVFE` | +| `managementUsername` | Management user with minimal permissions used for health checks | `management` | | `nodeSelector` | Node labels for pod assignment | `{}` | | `persistentVolume.accessMode` | Persistent volume access modes | `[ReadWriteOnce]` | | `persistentVolume.annotations` | Persistent volume annotations | `{}` | @@ -79,10 +82,11 @@ and their default values. | `persistentVolume.size` | Persistent volume size | `8Gi` | | `persistentVolume.storageClass` | Persistent volume storage class | `-` | | `podAntiAffinity` | Pod antiaffinity, `hard` or `soft` | `hard` | +| `policies` | HA policies to add to definitions.json | `` | | `prometheus.exporter.enabled` | Configures Prometheus Exporter to expose and scrape stats | `false` | | `prometheus.exporter.env` | Environment variables to set for Exporter container | `{}` | | `prometheus.exporter.image.repository` | Prometheus Exporter repository | `kbudde/rabbitmq-exporter` | -| `prometheus.exporter.image.tag` | Image Tag | `latest` | +| `prometheus.exporter.image.tag` | Image Tag | `v0.28.0` | | `prometheus.exporter.image.pullPolicy` | Image Pull Policy | `IfNotPresent` | | `prometheus.exporter.port` | Port Prometheus scrapes for metrics | `9090` | | `prometheus.exporter.capabilities` | Comma-separated list of extended scraping capabilities supported by the target RabbitMQ server. [Click here for details.](https://github.com/kbudde/rabbitmq_exporter#extended-rabbitmq-capabilities) | `bert,no_sort` | @@ -106,6 +110,7 @@ and their default values. | `rabbitmqMQTTPlugin.enabled` | Enable MQTT plugin | `false` | | `rabbitmqManagerPort` | RabbitMQ Manager port | `15672` | | `rabbitmqMemoryHighWatermark` | Memory high watermark | `256MB` | +| `rabbitmqMemoryHighWatermarkType` | Memory high watermark type. Either absolute or relative | `absolute` | | `rabbitmqNodePort` | Node port | `5672` | | `rabbitmqPassword` | RabbitMQ application password | _random 10 character long alphanumeric string_ | | `rabbitmqSTOMPPlugin.config` | STOMP configuration | `` | diff --git a/stable/rabbitmq-ha/templates/NOTES.txt b/stable/rabbitmq-ha/templates/NOTES.txt index 1aa1668850..b00c83deca 100644 --- a/stable/rabbitmq-ha/templates/NOTES.txt +++ b/stable/rabbitmq-ha/templates/NOTES.txt @@ -2,7 +2,7 @@ Credentials: - Username : {{ .Values.rabbitmqUsername }} + Username : {{ .Values.rabbitmqUsername -}} {{ if .Values.existingSecret }} Password : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.secretName" . }} -o jsonpath="{.data.rabbitmq-password}" | base64 --decode) ErLang Cookie : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.secretName" . }} -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 --decode) @@ -13,7 +13,7 @@ RabbitMQ can be accessed within the cluster on port {{ .Values.rabbitmqNodePort }} at {{ template "rabbitmq-ha.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local - To access for outside the cluster execute the following commands: + To access the cluster externally execute the following commands: {{- if contains "NodePort" .Values.service.type }} @@ -108,8 +108,3 @@ URL : http://127.0.0.1:{{ .Values.rabbitmqManagerPort }} {{- end }} - -To enable mirroring for all the host: - - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "rabbitmq-ha.name" . }}" -o jsonpath="{.items[0].metadata.name}") - kubectl exec $POD_NAME --namespace {{ .Release.Namespace }} -- rabbitmqctl set_policy ha-all "." '{"ha-mode":"all", "ha-sync-mode":"automatic"}' --apply-to all --priority 0 diff --git a/stable/rabbitmq-ha/templates/_helpers.tpl b/stable/rabbitmq-ha/templates/_helpers.tpl index 7fbce2c89a..815d76bbd7 100644 --- a/stable/rabbitmq-ha/templates/_helpers.tpl +++ b/stable/rabbitmq-ha/templates/_helpers.tpl @@ -48,3 +48,41 @@ Generate chart ssl secret name {{- define "rabbitmq-ha.certSecretName" -}} {{ default (print (include "rabbitmq-ha.fullname" .) "-cert") .Values.rabbitmqCert.existingSecret }} {{- end -}} + +{{/* +Defines a JSON file containing definitions of all broker objects (queues, exchanges, bindings, +users, virtual hosts, permissions and parameters) to load by the management plugin. +*/}} +{{- define "rabbitmq-ha.definitions" -}} +{ + "users": [ + { + "name": {{ .Values.managementUsername | quote }}, + "password": {{ .Values.managementPassword | quote }}, + "tags": "management" + }, + { + "name": {{ .Values.rabbitmqUsername | quote }}, + "password": {{ .Values.rabbitmqPassword | quote }}, + "tags": "administrator" + } + ], + "vhosts": [ + { + "name": {{ .Values.rabbitmqVhost | quote }} + } + ], + "permissions": [ + { + "user": {{ .Values.rabbitmqUsername | quote }}, + "vhost": {{ .Values.rabbitmqVhost | quote }}, + "configure": ".*", + "read": ".*", + "write": ".*" + } + ], + "policies": [ +{{ .Values.policies | indent 4 }} + ] +} +{{- end -}} diff --git a/stable/rabbitmq-ha/templates/alerts.yaml b/stable/rabbitmq-ha/templates/alerts.yaml index 756dacf736..79ad27d3ae 100644 --- a/stable/rabbitmq-ha/templates/alerts.yaml +++ b/stable/rabbitmq-ha/templates/alerts.yaml @@ -1,4 +1,4 @@ -{{ if and .Values.prometheus.operator.alerts.enabled .Values.prometheus.exporter.enabled }} +{{ if and .Values.prometheus.exporter.enabled .Values.prometheus.operator.enabled .Values.prometheus.operator.alerts.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/stable/rabbitmq-ha/templates/configmap.yaml b/stable/rabbitmq-ha/templates/configmap.yaml index 203b1f433a..710deef231 100644 --- a/stable/rabbitmq-ha/templates/configmap.yaml +++ b/stable/rabbitmq-ha/templates/configmap.yaml @@ -74,8 +74,10 @@ data: ## via a loopback interface (e.g. localhost) loopback_users.guest = false + management.load_definitions = /etc/definitions/definitions.json + ## Memory-based Flow Control threshold - vm_memory_high_watermark.absolute = {{ .Values.rabbitmqMemoryHighWatermark }} + vm_memory_high_watermark.{{ .Values.rabbitmqMemoryHighWatermarkType }} = {{ .Values.rabbitmqMemoryHighWatermark }} ## Auth HTTP Backend Plugin {{- if .Values.rabbitmqAuthHTTP.enabled }} @@ -112,4 +114,6 @@ data: {{ .Values.rabbitmqAmqpsSupport.config | indent 4 }} {{- end }} +{{ .Values.extraConfig | indent 4 }} + {{- end }} diff --git a/stable/rabbitmq-ha/templates/secret.yaml b/stable/rabbitmq-ha/templates/secret.yaml index 5bdfbf9ba3..763d47bef0 100644 --- a/stable/rabbitmq-ha/templates/secret.yaml +++ b/stable/rabbitmq-ha/templates/secret.yaml @@ -13,16 +13,12 @@ metadata: {{- end }} type: Opaque data: - {{ if .Values.rabbitmqPassword }} + {{- $password := .Values.rabbitmqPassword | default (randAlphaNum 24 | nospace) -}} + {{- $_ := set .Values "rabbitmqPassword" $password }} + rabbitmq-username: {{ .Values.rabbitmqUsername | b64enc | quote }} rabbitmq-password: {{ .Values.rabbitmqPassword | b64enc | quote }} - {{ else }} - rabbitmq-password: {{ randAlphaNum 24 | nospace | b64enc | quote }} - {{ end }} - {{ if .Values.rabbitmqErlangCookie }} - rabbitmq-erlang-cookie: {{ .Values.rabbitmqErlangCookie | b64enc | quote }} - {{ else }} - rabbitmq-erlang-cookie: {{ randAlphaNum 32 | b64enc | quote }} - {{ end }} + rabbitmq-erlang-cookie: {{ .Values.rabbitmqErlangCookie | default (randAlphaNum 32) | b64enc | quote }} + definitions.json: {{ include "rabbitmq-ha.definitions" . | b64enc | quote }} {{ end }} {{- if and .Values.rabbitmqCert.enabled (not .Values.rabbitmqCert.existingSecret) }} --- diff --git a/stable/rabbitmq-ha/templates/statefulset.yaml b/stable/rabbitmq-ha/templates/statefulset.yaml index 7b83c35153..68d92e60ec 100644 --- a/stable/rabbitmq-ha/templates/statefulset.yaml +++ b/stable/rabbitmq-ha/templates/statefulset.yaml @@ -97,21 +97,29 @@ spec: containerPort: 5671 {{- end }} livenessProbe: - exec: - command: - - rabbitmqctl - - status + httpGet: + path: /api/healthchecks/node + port: 15672 + scheme: HTTP + httpHeaders: + - name: Authorization + value: Basic {{ printf "%s:%s" .Values.managementUsername .Values.managementPassword | b64enc }} initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} failureThreshold: {{ .Values.livenessProbe.failureThreshold }} readinessProbe: - exec: - command: - - rabbitmqctl - - status + httpGet: + path: /api/healthchecks/node + port: 15672 + scheme: HTTP + httpHeaders: + - name: Authorization + value: Basic {{ printf "%s:%s" .Values.managementUsername .Values.managementPassword | b64enc }} initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} env: - name: MY_POD_NAME valueFrom: @@ -135,15 +143,6 @@ spec: - name: RABBITMQ_HIPE_COMPILE value: {{ .Values.rabbitmqHipeCompile | quote }} {{- end }} - - name: RABBITMQ_DEFAULT_USER - value: {{ .Values.rabbitmqUsername | quote }} - - name: RABBITMQ_DEFAULT_PASS - valueFrom: - secretKeyRef: - name: {{ template "rabbitmq-ha.secretName" . }} - key: rabbitmq-password - - name: RABBITMQ_DEFAULT_VHOST - value: {{ .Values.rabbitmqVhost | quote }} resources: {{ toYaml .Values.resources | indent 12 }} volumeMounts: @@ -151,6 +150,12 @@ spec: mountPath: /var/lib/rabbitmq - name: config mountPath: /etc/rabbitmq + readOnly: true + {{- if not .Values.existingSecret }} + - name: definitions + mountPath: /etc/definitions + readOnly: true + {{- end }} {{- if .Values.rabbitmqCert.enabled }} - name: cert mountPath: /etc/cert @@ -171,7 +176,10 @@ spec: value: "{{ .Values.prometheus.exporter.capabilities }}" {{- end }} - name: RABBIT_USER - value: {{ .Values.rabbitmqUsername | quote }} + valueFrom: + secretKeyRef: + name: {{ template "rabbitmq-ha.secretName" . }} + key: rabbitmq-username - name: RABBIT_PASSWORD valueFrom: secretKeyRef: @@ -219,6 +227,14 @@ spec: - name: configmap configMap: name: {{ template "rabbitmq-ha.fullname" . }} + {{- if not .Values.existingSecret }} + - name: definitions + secret: + secretName: {{ template "rabbitmq-ha.fullname" . }} + items: + - key: definitions.json + path: definitions.json + {{- end }} {{- if .Values.rabbitmqCert.enabled }} - name: cert secret: diff --git a/stable/rabbitmq-ha/values.yaml b/stable/rabbitmq-ha/values.yaml index e5a5a0b78c..be7651e4c4 100644 --- a/stable/rabbitmq-ha/values.yaml +++ b/stable/rabbitmq-ha/values.yaml @@ -4,6 +4,31 @@ rabbitmqUsername: guest # rabbitmqPassword: +## RabbitMQ Management user used for health checks +managementUsername: management +managementPassword: E9R3fjZm4ejFkVFE + +## Place any additional key/value configuration to add to rabbitmq.conf +## Ref: https://www.rabbitmq.com/configure.html#config-items +extraConfig: | +# queue_master_locator = min-masters + +## Sets the policies in definitions.json. This can be used to control the high +## availability of queues by mirroring them to multiple nodes. +## Ref: https://www.rabbitmq.com/ha.html +policies: |- +# { +# "name": "ha-all", +# "pattern": ".*", +# "vhost": "/", +# "definition": { +# "ha-mode": "all", +# "ha-sync-mode": "automatic", +# "ha-sync-batch-size": 1 +# } +# } + + ## RabbitMQ default VirtualHost ## Ref: https://www.rabbitmq.com/vhosts.html ## @@ -18,6 +43,7 @@ rabbitmqVhost: "/" ## Ref: http://www.rabbitmq.com/memory.html ## rabbitmqMemoryHighWatermark: 256MB +rabbitmqMemoryHighWatermarkType: absolute ## EPMD port for peer discovery service used by RabbitMQ nodes and CLI tools ## Ref: https://www.rabbitmq.com/clustering.html @@ -276,10 +302,12 @@ ingress: livenessProbe: initialDelaySeconds: 120 + periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 6 readinessProbe: + failureThreshold: 6 initialDelaySeconds: 10 timeoutSeconds: 3 periodSeconds: 5