From d6cdc8f09f9f413b36a471c9823d64a1b5e85994 Mon Sep 17 00:00:00 2001 From: Bastian Hofmann Date: Fri, 28 Jun 2019 19:31:38 +0200 Subject: [PATCH] [stable/redis] Improve liveness probe so that redis is not restarted while it is still loading its dataset into memory (#15093) On larger databases this can take more than just the initial probe delay. During this time redis will not answer the ping command with pong, but with loading. Of course the container should not be marked as ready, but it also should not restart which results in redis never starting successfully. Signed-off-by: Bastian Hofmann --- stable/redis/Chart.yaml | 2 +- stable/redis/templates/health-configmap.yaml | 66 +++++++++++++++---- .../templates/redis-master-statefulset.yaml | 6 +- .../templates/redis-slave-statefulset.yaml | 10 +-- 4 files changed, 64 insertions(+), 20 deletions(-) diff --git a/stable/redis/Chart.yaml b/stable/redis/Chart.yaml index 2ac0e68492..052e9b45df 100644 --- a/stable/redis/Chart.yaml +++ b/stable/redis/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: redis -version: 8.0.13 +version: 8.0.14 appVersion: 5.0.5 description: Open source, advanced key-value store. It is often referred to as a data structure server since keys can contain strings, hashes, lists, sets and sorted sets. keywords: diff --git a/stable/redis/templates/health-configmap.yaml b/stable/redis/templates/health-configmap.yaml index 6f0194fe23..9e255eb5da 100644 --- a/stable/redis/templates/health-configmap.yaml +++ b/stable/redis/templates/health-configmap.yaml @@ -8,7 +8,7 @@ metadata: release: {{ .Release.Name }} name: {{ template "redis.fullname" . }}-health data: - ping_local.sh: |- + ping_readiness_local.sh: |- {{- if .Values.usePasswordFile }} password_aux=`cat ${REDIS_PASSWORD_FILE}` export REDIS_PASSWORD=$password_aux @@ -27,6 +27,25 @@ data: echo "$response" exit 1 fi + ping_liveness_local.sh: |- +{{- if .Values.usePasswordFile }} + password_aux=`cat ${REDIS_PASSWORD_FILE}` + export REDIS_PASSWORD=$password_aux +{{- end }} + response=$( + timeout -s 9 $1 \ + redis-cli \ +{{- if .Values.usePassword }} + -a $REDIS_PASSWORD \ +{{- end }} + -h localhost \ + -p $REDIS_PORT \ + ping + ) + if [ "$response" != "PONG" ] && [ "$response" != "LOADING Redis is loading the dataset in memory" ]; then + echo "$response" + exit 1 + fi {{- if .Values.sentinel.enabled }} ping_sentinel.sh: |- {{- if .Values.usePasswordFile }} @@ -48,22 +67,22 @@ data: exit 1 fi parse_sentinels.awk: |- - /ip/ {FOUND_IP=1} - /port/ {FOUND_PORT=1} - /runid/ {FOUND_RUNID=1} - !/ip|port|runid/ { + /ip/ {FOUND_IP=1} + /port/ {FOUND_PORT=1} + /runid/ {FOUND_RUNID=1} + !/ip|port|runid/ { if (FOUND_IP==1) { IP=$1; FOUND_IP=0; - } + } else if (FOUND_PORT==1) { - PORT=$1; + PORT=$1; FOUND_PORT=0; } else if (FOUND_RUNID==1) { printf "\nsentinel known-sentinel {{ .Values.sentinel.masterSet }} %s %s %s", IP, PORT, $0; FOUND_RUNID=0; } } {{- end }} - ping_master.sh: |- + ping_readiness_master.sh: |- {{- if .Values.usePasswordFile }} password_aux=`cat ${REDIS_MASTER_PASSWORD_FILE}` export REDIS_MASTER_PASSWORD=$password_aux @@ -82,9 +101,34 @@ data: echo "$response" exit 1 fi - ping_local_and_master.sh: |- + ping_liveness_master.sh: |- +{{- if .Values.usePasswordFile }} + password_aux=`cat ${REDIS_MASTER_PASSWORD_FILE}` + export REDIS_MASTER_PASSWORD=$password_aux +{{- end }} + response=$( + timeout -s 9 $1 \ + redis-cli \ +{{- if .Values.usePassword }} + -a $REDIS_MASTER_PASSWORD \ +{{- end }} + -h $REDIS_MASTER_HOST \ + -p $REDIS_MASTER_PORT_NUMBER \ + ping + ) + if [ "$response" != "PONG" ] && [ "$response" != "LOADING Redis is loading the dataset in memory" ]; then + echo "$response" + exit 1 + fi + ping_readiness_local_and_master.sh: |- script_dir="$(dirname "$0")" exit_status=0 - "$script_dir/ping_local.sh" $1 || exit_status=$? - "$script_dir/ping_master.sh" $1 || exit_status=$? + "$script_dir/ping_readiness_local.sh" $1 || exit_status=$? + "$script_dir/ping_readiness_master.sh" $1 || exit_status=$? + exit $exit_status + ping_liveness_local_and_master.sh: |- + script_dir="$(dirname "$0")" + exit_status=0 + "$script_dir/ping_liveness_local.sh" $1 || exit_status=$? + "$script_dir/ping_liveness_master.sh" $1 || exit_status=$? exit $exit_status diff --git a/stable/redis/templates/redis-master-statefulset.yaml b/stable/redis/templates/redis-master-statefulset.yaml index 34534e58a9..69bd70e237 100644 --- a/stable/redis/templates/redis-master-statefulset.yaml +++ b/stable/redis/templates/redis-master-statefulset.yaml @@ -77,7 +77,7 @@ spec: fi if [[ ! -f /opt/bitnami/redis/etc/redis.conf ]];then cp /opt/bitnami/redis/mounted-etc/redis.conf /opt/bitnami/redis/etc/redis.conf - fi + fi ARGS=("--port" "${REDIS_PORT}") {{- if .Values.usePassword }} ARGS+=("--requirepass" "${REDIS_PASSWORD}") @@ -126,7 +126,7 @@ spec: command: - sh - -c - - /health/ping_local.sh {{ .Values.master.livenessProbe.timeoutSeconds }} + - /health/ping_liveness_local.sh {{ .Values.master.livenessProbe.timeoutSeconds }} {{- end }} {{- if .Values.master.readinessProbe.enabled}} readinessProbe: @@ -139,7 +139,7 @@ spec: command: - sh - -c - - /health/ping_local.sh {{ .Values.master.livenessProbe.timeoutSeconds }} + - /health/ping_readiness_local.sh {{ .Values.master.livenessProbe.timeoutSeconds }} {{- end }} resources: {{ toYaml .Values.master.resources | indent 10 }} diff --git a/stable/redis/templates/redis-slave-statefulset.yaml b/stable/redis/templates/redis-slave-statefulset.yaml index aae951c81a..da6b9228ed 100644 --- a/stable/redis/templates/redis-slave-statefulset.yaml +++ b/stable/redis/templates/redis-slave-statefulset.yaml @@ -89,7 +89,7 @@ spec: fi if [[ ! -f /opt/bitnami/redis/etc/redis.conf ]];then cp /opt/bitnami/redis/mounted-etc/redis.conf /opt/bitnami/redis/etc/redis.conf - fi + fi ARGS=("--port" "${REDIS_PORT}") ARGS+=("--slaveof" "${REDIS_MASTER_HOST}" "${REDIS_MASTER_PORT_NUMBER}") {{- if .Values.usePassword }} @@ -151,9 +151,9 @@ spec: - sh - -c {{- if .Values.sentinel.enabled }} - - /health/ping_local.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} + - /health/ping_liveness_local.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} {{- else }} - - /health/ping_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} + - /health/ping_liveness_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} {{- end }} {{- end }} @@ -169,9 +169,9 @@ spec: - sh - -c {{- if .Values.sentinel.enabled }} - - /health/ping_local.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} + - /health/ping_readiness_local.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} {{- else }} - - /health/ping_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} + - /health/ping_readiness_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} {{- end }} {{- end }} resources: