From 4fe7143f82c03a49b54193fda89cbc0cb360cd45 Mon Sep 17 00:00:00 2001 From: redixin Date: Tue, 8 Jan 2019 07:18:12 +0200 Subject: [PATCH] [incubator/etcd] Fix rejoin after failure (#9195) Signed-off-by: Sergey Skripnick --- incubator/etcd/Chart.yaml | 2 +- incubator/etcd/templates/statefulset.yaml | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/incubator/etcd/Chart.yaml b/incubator/etcd/Chart.yaml index 89f2bf27a4..14979d6e35 100755 --- a/incubator/etcd/Chart.yaml +++ b/incubator/etcd/Chart.yaml @@ -1,6 +1,6 @@ name: etcd home: https://github.com/coreos/etcd -version: 0.6.0 +version: 0.6.1 appVersion: 2.2.5 description: Distributed reliable key-value store for the most critical data of a distributed system. diff --git a/incubator/etcd/templates/statefulset.yaml b/incubator/etcd/templates/statefulset.yaml index d588a1aa64..1a856e5364 100644 --- a/incubator/etcd/templates/statefulset.yaml +++ b/incubator/etcd/templates/statefulset.yaml @@ -106,13 +106,23 @@ spec: etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 } + # we should wait for other pods to be up before trying to join + # otherwise we got "no such host" errors when trying to resolve other members + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + while true; do + echo "Waiting for ${SET_NAME}-${i}.${SET_NAME} to come up" + ping -W 1 -c 1 ${SET_NAME}-${i}.${SET_NAME} > /dev/null && break + sleep 1s + done + done + # re-joining after failure? if [ -e /var/run/etcd/default.etcd ]; then echo "Re-joining etcd member" member_id=$(cat /var/run/etcd/member_id) # re-join member - ETCDCTL_ENDPOINT=$(eps) etcdctl member update ${member_id} http://${HOSTNAME}.${SET_NAME}:2380 + ETCDCTL_ENDPOINT=$(eps) etcdctl member update ${member_id} http://${HOSTNAME}.${SET_NAME}:2380 | true exec etcd --name ${HOSTNAME} \ --listen-peer-urls http://0.0.0.0:2380 \ --listen-client-urls http://0.0.0.0:2379\ @@ -160,14 +170,6 @@ spec: --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE} fi - for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do - while true; do - echo "Waiting for ${SET_NAME}-${i}.${SET_NAME} to come up" - ping -W 1 -c 1 ${SET_NAME}-${i}.${SET_NAME} > /dev/null && break - sleep 1s - done - done - PEERS="" for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do PEERS="${PEERS}${PEERS:+,}${SET_NAME}-${i}=http://${SET_NAME}-${i}.${SET_NAME}:2380" @@ -207,4 +209,4 @@ spec: volumes: - name: datadir emptyDir: {} - {{- end }} \ No newline at end of file + {{- end }}