From f26d65ae2a795e5b28696bf87479342e0ebfaa13 Mon Sep 17 00:00:00 2001 From: Sergey Nuzhdin Date: Fri, 17 Mar 2017 23:18:26 +0100 Subject: [PATCH] Update incubator/etcd to statefulset (#631) (#660) * Update incubator/etcd to statefulset (#631) * typo fix * bump chart version * bump chart version again * bumped chart version --- incubator/etcd/Chart.yaml | 2 +- incubator/etcd/README.md | 30 +++++----- ...etcd-petset.yaml => etcd-statefulset.yaml} | 56 +++++++++---------- 3 files changed, 44 insertions(+), 44 deletions(-) rename incubator/etcd/templates/{etcd-petset.yaml => etcd-statefulset.yaml} (78%) diff --git a/incubator/etcd/Chart.yaml b/incubator/etcd/Chart.yaml index 006c9aea60..be4166e315 100755 --- a/incubator/etcd/Chart.yaml +++ b/incubator/etcd/Chart.yaml @@ -1,6 +1,6 @@ name: etcd home: https://github.com/coreos/etcd -version: 0.1.3 +version: 0.2.0 description: Distributed reliable key-value store for the most critical data of a distributed system. icon: https://raw.githubusercontent.com/coreos/etcd/master/logos/etcd-horizontal-color.png sources: diff --git a/incubator/etcd/README.md b/incubator/etcd/README.md index f1189524eb..1857f730bf 100644 --- a/incubator/etcd/README.md +++ b/incubator/etcd/README.md @@ -5,14 +5,14 @@ Credit to https://github.com/ingvagabund. This is an implementation of that work * https://github.com/kubernetes/contrib/pull/1295 ## Prerequisites Details -* Kubernetes 1.3 with alpha APIs enabled +* Kubernetes 1.5 (for `StatefulSets` support) * PV support on the underlying infrastructure -## PetSet Details -* http://kubernetes.io/docs/user-guide/petset/ +## StatefulSet Details +* https://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/ -## PetSet Caveats -* http://kubernetes.io/docs/user-guide/petset/#alpha-limitations +## StatefulSet Caveats +* https://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/#limitations ## Todo * Implement SSL @@ -20,7 +20,7 @@ Credit to https://github.com/ingvagabund. This is an implementation of that work ## Chart Details This chart will do the following: -* Implemented a dynamically scalable etcd cluster using Kubernetes PetSets +* Implemented a dynamically scalable etcd cluster using Kubernetes StatefulSets ## Installing the Chart @@ -41,7 +41,7 @@ The following tables lists the configurable parameters of the etcd chart and the | `Image` | Container image name | `gcr.io/google_containers/etcd-amd64` | | `ImageTag` | Container image tag | `2.2.5` | | `ImagePullPolicy` | Container pull policy | `Always` | -| `Replicas` | k8s petset replicas | `3` | +| `Replicas` | k8s statefulset replicas | `3` | | `Component` | k8s selector key | `etcd` | | `Cpu` | container requested cpu | `100m` | | `Memory` | container requested memory | `512Mi` | @@ -84,7 +84,7 @@ cluster is healthy ## Failover If any etcd member fails it gets re-joined eventually. -You can test the scenario by killing process of one of the pets: +You can test the scenario by killing process of one of the replicas: ```shell $ ps aux | grep etcd-1 @@ -137,7 +137,7 @@ etcd-0 1/1 Running 0 7m etcd-1 1/1 Running 0 7m etcd-2 1/1 Running 0 6m -$ kubectl patch petset/etcd -p '{"spec":{"replicas": 5}}' +$ kubectl patch statefulset/etcd -p '{"spec":{"replicas": 5}}' "etcd" patched $ kubectl get pods -l "component=${RELEASE-NAME}-etcd" @@ -149,11 +149,11 @@ etcd-3 1/1 Running 0 4s etcd-4 1/1 Running 0 1s ``` -Scaling-down is similar. For instance, changing the number of pets to ``4``: +Scaling-down is similar. For instance, changing the number of replicas to ``4``: ```sh -$ kubectl edit petset/etcd -petset "etcd" edited +$ kubectl edit statefulset/etcd +statefulset "etcd" edited $ kubectl get pods -l "component=${RELEASE-NAME}-etcd" NAME READY STATUS RESTARTS AGE @@ -163,7 +163,7 @@ etcd-2 1/1 Running 0 8m etcd-3 1/1 Running 0 4s ``` -Once a pet is terminated (either by running ``kubectl delete pod etcd-ID`` or scaling down), +Once a replica is terminated (either by running ``kubectl delete pod etcd-ID`` or scaling down), content of ``/var/run/etcd/`` directory is cleaned up. -If any of the etcd pets restarts (e.g. caused by etcd failure or any other), -the directory is kept untouched so the pet can recover from the failure. +If any of the etcd pods restarts (e.g. caused by etcd failure or any other), +the directory is kept untouched so the pod can recover from the failure. diff --git a/incubator/etcd/templates/etcd-petset.yaml b/incubator/etcd/templates/etcd-statefulset.yaml similarity index 78% rename from incubator/etcd/templates/etcd-petset.yaml rename to incubator/etcd/templates/etcd-statefulset.yaml index d53cd0481b..7b311198ff 100644 --- a/incubator/etcd/templates/etcd-petset.yaml +++ b/incubator/etcd/templates/etcd-statefulset.yaml @@ -20,8 +20,8 @@ spec: selector: component: "{{.Release.Name}}-{{.Values.Component}}" --- -apiVersion: apps/v1alpha1 -kind: PetSet +apiVersion: apps/v1beta1 +kind: StatefulSet metadata: name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" labels: @@ -59,7 +59,7 @@ spec: env: - name: INITIAL_CLUSTER_SIZE value: {{default 3 .Values.Replicas | quote }} - - name: PETSET_NAME + - name: SET_NAME value: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" volumeMounts: - name: datadir @@ -73,13 +73,13 @@ spec: - | EPS="" for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do - EPS="${EPS}${EPS:+,}http://${PETSET_NAME}-${i}.${PETSET_NAME}:2379" + EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379" done HOSTNAME=$(hostname) member_hash() { - etcdctl member list | grep http://${HOSTNAME}.${PETSET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 + etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 } echo "Removing ${HOSTNAME} from etcd cluster" @@ -98,20 +98,20 @@ spec: # store member id into PVC for later member replacement collect_member() { while ! etcdctl member list &>/dev/null; do sleep 1; done - etcdctl member list | grep http://${HOSTNAME}.${PETSET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 > /var/run/etcd/member_id + etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 > /var/run/etcd/member_id exit 0 } eps() { EPS="" for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do - EPS="${EPS}${EPS:+,}http://${PETSET_NAME}-${i}.${PETSET_NAME}:2379" + EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379" done echo ${EPS} } member_hash() { - etcdctl member list | grep http://${HOSTNAME}.${PETSET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 + etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 } # re-joining after failure? @@ -120,19 +120,19 @@ spec: member_id=$(cat /var/run/etcd/member_id) # re-join member - ETCDCTL_ENDPOINT=$(eps) etcdctl member update ${member_id} http://${HOSTNAME}.${PETSET_NAME}:2380 + ETCDCTL_ENDPOINT=$(eps) etcdctl member update ${member_id} http://${HOSTNAME}.${SET_NAME}:2380 exec etcd --name ${HOSTNAME} \ - --listen-peer-urls http://${HOSTNAME}.${PETSET_NAME}:2380 \ - --listen-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379,http://127.0.0.1:2379 \ - --advertise-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379 \ + --listen-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --listen-client-urls http://${HOSTNAME}.${SET_NAME}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \ --data-dir /var/run/etcd/default.etcd fi - # etcd-PET_ID - PET_ID=${HOSTNAME:5:${#HOSTNAME}} + # etcd-SET_ID + SET_ID=${HOSTNAME:5:${#HOSTNAME}} - # adding a new member to existing cluster (assuming all initial pets are available) - if [ "${PET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then + # adding a new member to existing cluster (assuming all initial pods are available) + if [ "${SET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then export ETCDCTL_ENDPOINT=$(eps) # member already added? @@ -145,7 +145,7 @@ spec: fi echo "Adding new member" - etcdctl member add ${HOSTNAME} http://${HOSTNAME}.${PETSET_NAME}:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs + etcdctl member add ${HOSTNAME} http://${HOSTNAME}.${SET_NAME}:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs if [ $? -ne 0 ]; then echo "Exiting" @@ -159,36 +159,36 @@ spec: collect_member & exec etcd --name ${HOSTNAME} \ - --listen-peer-urls http://${HOSTNAME}.${PETSET_NAME}:2380 \ - --listen-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379,http://127.0.0.1:2379 \ - --advertise-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379 \ + --listen-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --listen-client-urls http://${HOSTNAME}.${SET_NAME}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \ --data-dir /var/run/etcd/default.etcd \ - --initial-advertise-peer-urls http://${HOSTNAME}.${PETSET_NAME}:2380 \ + --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ --initial-cluster ${ETCD_INITIAL_CLUSTER} \ --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE} fi for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do while true; do - echo "Waiting for ${PETSET_NAME}-${i}.${PETSET_NAME} to come up" - ping -W 1 -c 1 ${PETSET_NAME}-${i}.${PETSET_NAME} > /dev/null && break + echo "Waiting for ${SET_NAME}-${i}.${SET_NAME} to come up" + ping -W 1 -c 1 ${SET_NAME}-${i}.${SET_NAME} > /dev/null && break sleep 1s done done PEERS="" for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do - PEERS="${PEERS}${PEERS:+,}${PETSET_NAME}-${i}=http://${PETSET_NAME}-${i}.${PETSET_NAME}:2380" + PEERS="${PEERS}${PEERS:+,}${SET_NAME}-${i}=http://${SET_NAME}-${i}.${SET_NAME}:2380" done collect_member & # join member exec etcd --name ${HOSTNAME} \ - --initial-advertise-peer-urls http://${HOSTNAME}.${PETSET_NAME}:2380 \ - --listen-peer-urls http://${HOSTNAME}.${PETSET_NAME}:2380 \ - --listen-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379,http://127.0.0.1:2379 \ - --advertise-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379 \ + --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --listen-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --listen-client-urls http://${HOSTNAME}.${SET_NAME}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \ --initial-cluster-token etcd-cluster-1 \ --initial-cluster ${PEERS} \ --initial-cluster-state new \