diff --git a/incubator/etcd/.helmignore b/incubator/etcd/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/incubator/etcd/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/incubator/etcd/Chart.yaml b/incubator/etcd/Chart.yaml new file mode 100755 index 0000000000..f3b1e25aae --- /dev/null +++ b/incubator/etcd/Chart.yaml @@ -0,0 +1,9 @@ +name: etcd +home: https://github.com/coreos/etcd +version: 0.1.0 +description: Etcd Helm chart for Kubernetes. +sources: + - https://github.com/coreos/etcd +maintainers: + - name: Lachlan Evenson + email: lachlan@deis.com diff --git a/incubator/etcd/README.md b/incubator/etcd/README.md new file mode 100644 index 0000000000..b80532a920 --- /dev/null +++ b/incubator/etcd/README.md @@ -0,0 +1,178 @@ +# Etcd Helm Chart + +Credit to https://github.com/ingvagabund. This is an implementation of that work + +* https://github.com/kubernetes/contrib/pull/1295 + +## Prerequisites Details +* Kubernetes 1.3 with alpha APIs enable +* PV support on the underlying infrastructure + +## PetSet Details +* http://kubernetes.io/docs/user-guide/petset/ + +## PetSet Caveats +* http://kubernetes.io/docs/user-guide/petset/#alpha-limitations + +## Todo +* Implement SSL + +## Chart Details +This chart will do the following: + +* Implemented a dynamically scalable etcd cluster using Kubernetes PetSets + +## Get this chart + +Download the latest release of the chart from the [releases](../../../releases) page. + +Alternatively, clone the repo if you wish to use the development snapshot: + +```bash +$ git clone https://github.com/kubernetes/charts.git +``` + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```bash +$ helm install --name my-release etcd-x.x.x.tgz +``` + +## Configuration + +The following tables lists the configurable parameters of the etcd chart and their default values. + +| Parameter | Description | Default | +|-----------------------|----------------------------------|----------------------------------------------------------| +| `Name` | Spark master name | `etcd` | +| `Image` | Container image name | `gcr.io/google_containers/etcd-amd64` | +| `ImageTag` | Container image tag | `2.2.5` | +| `ImagePullPolicy` | Container pull policy | `Always` | +| `Replicas` | k8s petset replicas | `3` | +| `Component` | k8s selector key | `etcd` | +| `Cpu` | container requested cpu | `100m` | +| `Memory` |container requested memory | `512Mi` | +| `ClientPort` | k8s service port | `2379` | +| `PeerPorts`| Container listening port | `2380` | +| `Storage`| Persistent volume size | `1Gi` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```bash +$ helm install --name my-release -f values.yaml etcd-x.x.x.tgz +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +# Deep dive + +## Cluster Health + +``` +$ for i in <0..n>; do kubectl exec -- sh -c 'etcdctl cluster-health'; done +``` +eg. +``` +$ for i in {0..9}; do kubectl exec named-lynx-etcd-$i --namespace=etcd -- sh -c 'etcdctl cluster-health'; done +member 7878c44dabe58db is healthy: got healthy result from http://named-lynx-etcd-7.named-lynx-etcd:2379 +member 19d2ab7b415341cc is healthy: got healthy result from http://named-lynx-etcd-4.named-lynx-etcd:2379 +member 6b627d1b92282322 is healthy: got healthy result from http://named-lynx-etcd-3.named-lynx-etcd:2379 +member 6bb377156d9e3fb3 is healthy: got healthy result from http://named-lynx-etcd-0.named-lynx-etcd:2379 +member 8ebbb00c312213d6 is healthy: got healthy result from http://named-lynx-etcd-8.named-lynx-etcd:2379 +member a32e3e8a520ff75f is healthy: got healthy result from http://named-lynx-etcd-5.named-lynx-etcd:2379 +member dc83003f0a226816 is healthy: got healthy result from http://named-lynx-etcd-2.named-lynx-etcd:2379 +member e3dc94686f60465d is healthy: got healthy result from http://named-lynx-etcd-6.named-lynx-etcd:2379 +member f5ee1ca177a88a58 is healthy: got healthy result from http://named-lynx-etcd-1.named-lynx-etcd:2379 +cluster is healthy +``` + +## Failover + +If any etcd member fails it gets re-joined eventually. +You can test the scenario by killing process of one of the pets: + +```shell +$ ps aux | grep etcd-1 +$ kill -9 ETCD_1_PID +``` + +```shell +$ kubectl get pods -l "app=etcd" +NAME READY STATUS RESTARTS AGE +etcd-0 1/1 Running 0 54s +etcd-2 1/1 Running 0 51s +``` + +After a while: + +```shell +$ kubectl get pods -l "app=etcd" +NAME READY STATUS RESTARTS AGE +etcd-0 1/1 Running 0 1m +etcd-1 1/1 Running 0 20s +etcd-2 1/1 Running 0 1m +``` + +You can check state of re-joining from ``etcd-1``'s logs: + +```shell +$ kubectl logs etcd-1 +Waiting for etcd-0.etcd to come up +Waiting for etcd-1.etcd to come up +ping: bad address 'etcd-1.etcd' +Waiting for etcd-1.etcd to come up +Waiting for etcd-2.etcd to come up +Re-joining etcd member +Updated member with ID 7fd61f3f79d97779 in cluster +2016-06-20 11:04:14.962169 I | etcdmain: etcd Version: 2.2.5 +2016-06-20 11:04:14.962287 I | etcdmain: Git SHA: bc9ddf2 +... +``` + +## Scaling using kubectl + +This is for reference. Scaling should be managed by `helm upgrade` + +The etcd cluster can be scale up by running ``kubectl patch`` or ``kubectl edit``. For instance, + +```sh +$ kubectl get pods -l "app=etcd" +NAME READY STATUS RESTARTS AGE +etcd-0 1/1 Running 0 7m +etcd-1 1/1 Running 0 7m +etcd-2 1/1 Running 0 6m + +$ kubectl patch petset/etcd -p '{"spec":{"replicas": 5}}' +"etcd" patched + +$ kubectl get pods -l "app=etcd" +NAME READY STATUS RESTARTS AGE +etcd-0 1/1 Running 0 8m +etcd-1 1/1 Running 0 8m +etcd-2 1/1 Running 0 8m +etcd-3 1/1 Running 0 4s +etcd-4 1/1 Running 0 1s +``` + +Scaling-down is similar. For instance, changing the number of pets to ``4``: + +```sh +$ kubectl edit petset/etcd +petset "etcd" edited + +$ kubectl get pods -l "app=etcd" +NAME READY STATUS RESTARTS AGE +etcd-0 1/1 Running 0 8m +etcd-1 1/1 Running 0 8m +etcd-2 1/1 Running 0 8m +etcd-3 1/1 Running 0 4s +``` + +Once a pet is terminated (either by running ``kubectl delete pod etcd-ID`` or scaling down), +content of ``/var/run/etcd/`` directory is cleaned up. +If any of the etcd pets restarts (e.g. caused by etcd failure or any other), +the directory is kept untouched so the pet can recover from the failure. diff --git a/incubator/etcd/templates/etcd-petset.yaml b/incubator/etcd/templates/etcd-petset.yaml new file mode 100644 index 0000000000..160f920f39 --- /dev/null +++ b/incubator/etcd/templates/etcd-petset.yaml @@ -0,0 +1,212 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +metadata: + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + component: "{{.Release.Name}}-{{.Values.Component}}" + annotations: + "helm.sh/created": {{.Release.Time.Seconds | quote }} +spec: + ports: + - port: {{.Values.PeerPort}} + name: peer + - port: {{.Values.ClientPort}} + name: client + clusterIP: None + selector: + component: "{{.Release.Name}}-{{.Values.Component}}" +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + component: "{{.Release.Name}}-{{.Values.Component}}" + annotations: + "helm.sh/created": {{.Release.Time.Seconds | quote }} +spec: + serviceName: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" + replicas: {{default 3 .Values.Replicas | quote }} + template: + metadata: + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + component: "{{.Release.Name}}-{{.Values.Component}}" + annotations: + "helm.sh/created": {{.Release.Time.Seconds | quote }} + pod.alpha.kubernetes.io/initialized: "true" + spec: + containers: + - name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" + image: "{{.Values.Image}}:{{.Values.ImageTag}}" + imagePullPolicy: "{{.Values.ImagePullPolicy}}" + ports: + - containerPort: {{.Values.PeerPort}} + name: peer + - containerPort: {{.Values.ClientPort}} + name: client + resources: + requests: + cpu: "{{.Values.Cpu}}" + memory: "{{.Values.Memory}}" + env: + - name: INITIAL_CLUSTER_SIZE + value: {{default 3 .Values.Replicas | quote }} + - name: PETSET_NAME + value: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" + volumeMounts: + - name: datadir + mountPath: /var/run/etcd + lifecycle: + preStop: + exec: + command: + - "/bin/sh" + - "-ec" + - | + EPS="" + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + EPS="${EPS}${EPS:+,}http://${PETSET_NAME}-${i}.${PETSET_NAME}:2379" + done + + HOSTNAME=$(hostname) + + member_hash() { + etcdctl member list | grep http://${HOSTNAME}.${PETSET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 + } + + echo "Removing ${HOSTNAME} from etcd cluster" + + ETCDCTL_ENDPOINT=${EPS} etcdctl member remove $(member_hash) + if [ $? -eq 0 ]; then + # Remove everything otherwise the cluster will no longer scale-up + rm -rf /var/run/etcd/* + fi + command: + - "/bin/sh" + - "-ec" + - | + HOSTNAME=$(hostname) + + # store member id into PVC for later member replacement + collect_member() { + while ! etcdctl member list &>/dev/null; do sleep 1; done + etcdctl member list | grep ${HOSTNAME}.etcd | cut -d':' -f1 | cut -d'[' -f1 > /var/run/etcd/member_id + exit 0 + } + + eps() { + EPS="" + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + EPS="${EPS}${EPS:+,}http://${PETSET_NAME}-${i}.${PETSET_NAME}:2379" + done + echo ${EPS} + } + + member_hash() { + etcdctl member list | grep http://${HOSTNAME}.${PETSET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 + } + + # re-joining after failure? + if [ -e /var/run/etcd/default.etcd ]; then + echo "Re-joining etcd member" + member_id=$(cat /var/run/etcd/member_id) + + # re-join member + ETCDCTL_ENDPOINT=$(eps) etcdctl member update ${member_id} http://${HOSTNAME}.${PETSET_NAME}:2380 + exec etcd --name ${HOSTNAME} \ + --listen-peer-urls http://${HOSTNAME}.${PETSET_NAME}:2380 \ + --listen-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379 \ + --data-dir /var/run/etcd/default.etcd + fi + + # etcd-PET_ID + PET_ID=${HOSTNAME:5:${#HOSTNAME}} + + # adding a new member to existing cluster (assuming all initial pets are available) + if [ "${PET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then + export ETCDCTL_ENDPOINT=$(eps) + + # member already added? + MEMBER_HASH=$(member_hash) + if [ -n "${MEMBER_HASH}" ]; then + # the member hash exists but for some reason etcd failed + # as the datadir has not be created, we can remove the member + # and retrieve new hash + etcdctl member remove ${MEMBER_HASH} + fi + + echo "Adding new member" + etcdctl member add ${HOSTNAME} http://${HOSTNAME}.${PETSET_NAME}:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs + + if [ $? -ne 0 ]; then + echo "Exiting" + rm -f /var/run/etcd/new_member_envs + exit 1 + fi + + cat /var/run/etcd/new_member_envs + source /var/run/etcd/new_member_envs + + collect_member & + + exec etcd --name ${HOSTNAME} \ + --listen-peer-urls http://${HOSTNAME}.${PETSET_NAME}:2380 \ + --listen-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379 \ + --data-dir /var/run/etcd/default.etcd \ + --initial-advertise-peer-urls http://${HOSTNAME}.${PETSET_NAME}:2380 \ + --initial-cluster ${ETCD_INITIAL_CLUSTER} \ + --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE} + fi + + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + while true; do + echo "Waiting for ${PETSET_NAME}-${i}.${PETSET_NAME} to come up" + ping -W 1 -c 1 ${PETSET_NAME}-${i}.${PETSET_NAME} > /dev/null && break + sleep 1s + done + done + + PEERS="" + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + PEERS="${PEERS}${PEERS:+,}${PETSET_NAME}-${i}=http://${PETSET_NAME}-${i}.${PETSET_NAME}:2380" + done + + collect_member & + + # join member + exec etcd --name ${HOSTNAME} \ + --initial-advertise-peer-urls http://${HOSTNAME}.${PETSET_NAME}:2380 \ + --listen-peer-urls http://${HOSTNAME}.${PETSET_NAME}:2380 \ + --listen-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls http://${HOSTNAME}.${PETSET_NAME}:2379 \ + --initial-cluster-token etcd-cluster-1 \ + --initial-cluster ${PEERS} \ + --initial-cluster-state new \ + --data-dir /var/run/etcd/default.etcd + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + # upstream recommended max is 700M + storage: "{{.Values.Storage}}" diff --git a/incubator/etcd/values.yaml b/incubator/etcd/values.yaml new file mode 100644 index 0000000000..57bf923612 --- /dev/null +++ b/incubator/etcd/values.yaml @@ -0,0 +1,16 @@ +# Default values for etcd. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +Name: etcd +PeerPort: 2380 +ClientPort: 2379 +Component: "etcd" +Replicas: 3 +Image: "gcr.io/google_containers/etcd-amd64" +ImageTag: "2.2.5" +ImagePullPolicy: "Always" +Cpu: "100m" +Memory: "512Mi" +Storage: "1Gi"