[stable/redis] Improve liveness probe so that redis is not restarted while it is still loading its dataset into memory (#15093)

On larger databases this can take more than just the initial probe delay. During this time redis will not answer the ping command with pong, but with loading. Of course the container should not be marked as ready, but it also should not restart which results in redis never starting successfully.

Signed-off-by: Bastian Hofmann <bashofmann@gmail.com>
This commit is contained in:
Bastian Hofmann
2019-06-28 10:31:38 -07:00
committed by Kubernetes Prow Robot
parent 9bb2eeb525
commit d6cdc8f09f
4 changed files with 64 additions and 20 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
apiVersion: v1
name: redis
version: 8.0.13
version: 8.0.14
appVersion: 5.0.5
description: Open source, advanced key-value store. It is often referred to as a data structure server since keys can contain strings, hashes, lists, sets and sorted sets.
keywords:
+55 -11
View File
@@ -8,7 +8,7 @@ metadata:
release: {{ .Release.Name }}
name: {{ template "redis.fullname" . }}-health
data:
ping_local.sh: |-
ping_readiness_local.sh: |-
{{- if .Values.usePasswordFile }}
password_aux=`cat ${REDIS_PASSWORD_FILE}`
export REDIS_PASSWORD=$password_aux
@@ -27,6 +27,25 @@ data:
echo "$response"
exit 1
fi
ping_liveness_local.sh: |-
{{- if .Values.usePasswordFile }}
password_aux=`cat ${REDIS_PASSWORD_FILE}`
export REDIS_PASSWORD=$password_aux
{{- end }}
response=$(
timeout -s 9 $1 \
redis-cli \
{{- if .Values.usePassword }}
-a $REDIS_PASSWORD \
{{- end }}
-h localhost \
-p $REDIS_PORT \
ping
)
if [ "$response" != "PONG" ] && [ "$response" != "LOADING Redis is loading the dataset in memory" ]; then
echo "$response"
exit 1
fi
{{- if .Values.sentinel.enabled }}
ping_sentinel.sh: |-
{{- if .Values.usePasswordFile }}
@@ -48,22 +67,22 @@ data:
exit 1
fi
parse_sentinels.awk: |-
/ip/ {FOUND_IP=1}
/port/ {FOUND_PORT=1}
/runid/ {FOUND_RUNID=1}
!/ip|port|runid/ {
/ip/ {FOUND_IP=1}
/port/ {FOUND_PORT=1}
/runid/ {FOUND_RUNID=1}
!/ip|port|runid/ {
if (FOUND_IP==1) {
IP=$1; FOUND_IP=0;
}
}
else if (FOUND_PORT==1) {
PORT=$1;
PORT=$1;
FOUND_PORT=0;
} else if (FOUND_RUNID==1) {
printf "\nsentinel known-sentinel {{ .Values.sentinel.masterSet }} %s %s %s", IP, PORT, $0; FOUND_RUNID=0;
}
}
{{- end }}
ping_master.sh: |-
ping_readiness_master.sh: |-
{{- if .Values.usePasswordFile }}
password_aux=`cat ${REDIS_MASTER_PASSWORD_FILE}`
export REDIS_MASTER_PASSWORD=$password_aux
@@ -82,9 +101,34 @@ data:
echo "$response"
exit 1
fi
ping_local_and_master.sh: |-
ping_liveness_master.sh: |-
{{- if .Values.usePasswordFile }}
password_aux=`cat ${REDIS_MASTER_PASSWORD_FILE}`
export REDIS_MASTER_PASSWORD=$password_aux
{{- end }}
response=$(
timeout -s 9 $1 \
redis-cli \
{{- if .Values.usePassword }}
-a $REDIS_MASTER_PASSWORD \
{{- end }}
-h $REDIS_MASTER_HOST \
-p $REDIS_MASTER_PORT_NUMBER \
ping
)
if [ "$response" != "PONG" ] && [ "$response" != "LOADING Redis is loading the dataset in memory" ]; then
echo "$response"
exit 1
fi
ping_readiness_local_and_master.sh: |-
script_dir="$(dirname "$0")"
exit_status=0
"$script_dir/ping_local.sh" $1 || exit_status=$?
"$script_dir/ping_master.sh" $1 || exit_status=$?
"$script_dir/ping_readiness_local.sh" $1 || exit_status=$?
"$script_dir/ping_readiness_master.sh" $1 || exit_status=$?
exit $exit_status
ping_liveness_local_and_master.sh: |-
script_dir="$(dirname "$0")"
exit_status=0
"$script_dir/ping_liveness_local.sh" $1 || exit_status=$?
"$script_dir/ping_liveness_master.sh" $1 || exit_status=$?
exit $exit_status
@@ -77,7 +77,7 @@ spec:
fi
if [[ ! -f /opt/bitnami/redis/etc/redis.conf ]];then
cp /opt/bitnami/redis/mounted-etc/redis.conf /opt/bitnami/redis/etc/redis.conf
fi
fi
ARGS=("--port" "${REDIS_PORT}")
{{- if .Values.usePassword }}
ARGS+=("--requirepass" "${REDIS_PASSWORD}")
@@ -126,7 +126,7 @@ spec:
command:
- sh
- -c
- /health/ping_local.sh {{ .Values.master.livenessProbe.timeoutSeconds }}
- /health/ping_liveness_local.sh {{ .Values.master.livenessProbe.timeoutSeconds }}
{{- end }}
{{- if .Values.master.readinessProbe.enabled}}
readinessProbe:
@@ -139,7 +139,7 @@ spec:
command:
- sh
- -c
- /health/ping_local.sh {{ .Values.master.livenessProbe.timeoutSeconds }}
- /health/ping_readiness_local.sh {{ .Values.master.livenessProbe.timeoutSeconds }}
{{- end }}
resources:
{{ toYaml .Values.master.resources | indent 10 }}
@@ -89,7 +89,7 @@ spec:
fi
if [[ ! -f /opt/bitnami/redis/etc/redis.conf ]];then
cp /opt/bitnami/redis/mounted-etc/redis.conf /opt/bitnami/redis/etc/redis.conf
fi
fi
ARGS=("--port" "${REDIS_PORT}")
ARGS+=("--slaveof" "${REDIS_MASTER_HOST}" "${REDIS_MASTER_PORT_NUMBER}")
{{- if .Values.usePassword }}
@@ -151,9 +151,9 @@ spec:
- sh
- -c
{{- if .Values.sentinel.enabled }}
- /health/ping_local.sh {{ .Values.slave.livenessProbe.timeoutSeconds }}
- /health/ping_liveness_local.sh {{ .Values.slave.livenessProbe.timeoutSeconds }}
{{- else }}
- /health/ping_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }}
- /health/ping_liveness_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }}
{{- end }}
{{- end }}
@@ -169,9 +169,9 @@ spec:
- sh
- -c
{{- if .Values.sentinel.enabled }}
- /health/ping_local.sh {{ .Values.slave.livenessProbe.timeoutSeconds }}
- /health/ping_readiness_local.sh {{ .Values.slave.livenessProbe.timeoutSeconds }}
{{- else }}
- /health/ping_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }}
- /health/ping_readiness_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }}
{{- end }}
{{- end }}
resources: