From a911c0bc412d4d770e33b71d2e42f2bfda79a2a2 Mon Sep 17 00:00:00 2001 From: Dhilip Date: Mon, 22 May 2017 20:13:09 +0530 Subject: [PATCH] redis-cache chart (#917) * Initial Checkin of redis-cache chart * Fix review comments * fix review comments * re-org values.yaml based on components * Fix typos and more improvemtns to values.yaml * Address Additional rounds of review comments * Update Sources, icon and image tags * update description --- incubator/redis-cache/.helmignore | 21 ++++ incubator/redis-cache/Chart.yaml | 11 ++ incubator/redis-cache/README.md | 119 +++++++++++++++++++ incubator/redis-cache/templates/NOTES.txt | 2 + incubator/redis-cache/templates/_helpers.tpl | 16 +++ incubator/redis-cache/templates/pdb.yaml | 12 ++ incubator/redis-cache/templates/service.yaml | 18 +++ incubator/redis-cache/templates/ss.yaml | 88 ++++++++++++++ incubator/redis-cache/values.yaml | 49 ++++++++ 9 files changed, 336 insertions(+) create mode 100644 incubator/redis-cache/.helmignore create mode 100644 incubator/redis-cache/Chart.yaml create mode 100644 incubator/redis-cache/README.md create mode 100644 incubator/redis-cache/templates/NOTES.txt create mode 100644 incubator/redis-cache/templates/_helpers.tpl create mode 100644 incubator/redis-cache/templates/pdb.yaml create mode 100644 incubator/redis-cache/templates/service.yaml create mode 100644 incubator/redis-cache/templates/ss.yaml create mode 100644 incubator/redis-cache/values.yaml diff --git a/incubator/redis-cache/.helmignore b/incubator/redis-cache/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/incubator/redis-cache/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/incubator/redis-cache/Chart.yaml b/incubator/redis-cache/Chart.yaml new file mode 100644 index 0000000000..1284637da2 --- /dev/null +++ b/incubator/redis-cache/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +description: A pure in-memory redis cache, using statefulset and redis-sentinel-micro +name: redis-cache +version: 0.1.0 +icon: https://redis.io/images/redis-white.png +sources: + - https://github.com/antirez/redis + - https://github.com/dhilipkumars/redis-sentinel-micro/tree/k8s +maintainers: + - name: dhilipkumars + email: dhilip.kumar.s@huawei.com diff --git a/incubator/redis-cache/README.md b/incubator/redis-cache/README.md new file mode 100644 index 0000000000..47d9af4fd3 --- /dev/null +++ b/incubator/redis-cache/README.md @@ -0,0 +1,119 @@ +## Redis-cache + +This chart will allow us to use redis as a pure in-memory cache, this uses a special utility to perform master-slave promotion which overcomes this [problem](https://redis.io/topics/replication#safety-of-replication-when-master-has-persistence-turned-off), K8s could restart the master way before it is detected and agreed among the redis-sentinels. + +###Prerequisites +* Kubernetes 1.6+ + +Install Redis-cache chart as an in-memory cache. +* Uses 7MB [redis:3-alpine](https://hub.docker.com/r/library/redis/tags/3-alpine/) image which is extremely light weight +* Uses [redis-sentinel-k8s](https://github.com/dhilipkumars/redis-sentinel-micro/tree/k8s) for automatic slave promotion if master fails, feel free to fork. +* Super fast caching as no persistence involved +* Has Pod Disruption Budget +* Uses Statefulset +* Has Anti-Affinity Configured + +Quick Benchmark comparison between redis with and without persistence enabled, as you can notice the writes are at least 3 times slower. + +Redis persistence enabled +``` +$ k exec -i -t ${REPLICA-NAME} -c redis -- redis-benchmark -q -h ${REPLICA-NAME}.${POD-NAME}.${NAMESPACE} -p 6379 -t set,get -n 100000 -d 100 -r 1000000 +SET: 22026.43 requests per second +GET: 76161.46 requests per second +``` +Redis without persistence +``` +$ k exec -i -t ${REPLICA-NAME} -c redis -- redis-benchmark -q -h ${REPLICA-NAME}.${POD-NAME}.${NAMESPACE} -p 6379 -t set,get -n 100000 -d 100 -r 1000000 +SET: 60060.06 requests per second +GET: 89285.71 requests per second +``` + +Here is a sample demo of master slave promotion. + +Install this chart +``` +$helm install -n dev incubator/redis-cache/ +NAME: dev +LAST DEPLOYED: Wed Apr 12 23:25:56 2017 +NAMESPACE: default +STATUS: DEPLOYED + +RESOURCES: +==> v1/Service +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rd-dev None 6379/TCP 0s + +==> policy/v1beta1/PodDisruptionBudget +NAME MIN-AVAILABLE ALLOWED-DISRUPTIONS AGE +rd-dev 2 0 0s + +==> apps/v1beta1/StatefulSet +NAME DESIRED CURRENT AGE +rd-dev 3 0 0s + + +NOTES: +1. To Find which replica is the master + kubectl exec -i -t rd-dev-0 -c redis -- redis-cli -h rd-dev-0.rd-dev.default -p 6379 info replication +``` + +Wait until the PODs are created and check which replica is the master (by default it creates 3 replicas with 1 master and 2 slaves) +``` +$kubectl get pods +NAME READY STATUS RESTARTS AGE +rd-dev-0 2/2 Running 0 57s +rd-dev-1 2/2 Running 0 41s +rd-dev-2 2/2 Running 0 29s + +$kubectl exec -i -t rd-dev-0 -c redis -- redis-cli -h rd-dev-0.rd-dev.default -p 6379 info replication +# Replication +role:master +connected_slaves:2 +slave0:ip=10.244.2.18,port=6379,state=online,offset=29,lag=0 +slave1:ip=10.244.1.16,port=6379,state=online,offset=29,lag=0 +master_repl_offset:29 +repl_backlog_active:1 +repl_backlog_size:1048576 +repl_backlog_first_byte_offset:2 +repl_backlog_histlen:28 +``` + +Set a Key +``` +$kubectl exec -i -t rd-dev-0 -c redis -- redis-cli -h rd-dev-0.rd-dev.default -p 6379 set foo bar +OK +``` + +Check if that is replicated in any of the slaves +``` +$kubectl exec -i -t rd-dev-0 -c redis -- redis-cli -h rd-dev-2.rd-dev.default -p 6379 get foo +"bar" +``` + +Kill the master +``` +$kubeclt delete po/rd-dev-0 +pod "rd-dev-0" deleted +``` + +Wait until the kubernetes restarts rd-dev-0, once this is up you can check sentinel-micro's logs to find out which of the slave is now elected as the new master. +``` +$kubectl logs rd-dev-0 sentinel-micro +I0412 18:07:08.633263 1 redis_sentinel_k8s.go:368] Available endpoints are [rd-dev-2.rd-dev.default.svc.cluster.local rd-dev-1.rd-dev.default.svc.cluster.local] +I0412 18:07:08.633392 1 redis_sentinel_k8s.go:194] Processing rd-dev-2.rd-dev.default.svc.cluster.local +I0412 18:07:08.633446 1 redis_sentinel_k8s.go:194] Processing rd-dev-1.rd-dev.default.svc.cluster.local +R=&{ slave -1 7 937 rd-dev-0.rd-dev.default 6379 100 false } +R=&{ slave -1 7 937 rd-dev-0.rd-dev.default 6379 100 false } +I0412 18:07:08.645222 1 redis_sentinel_k8s.go:382] OldMaster= NewMaster=&{rd-dev-1.rd-dev.default:6379 slave -1 7 937 rd-dev-0.rd-dev.default 6379 100 false 0xc42000e300} +I0412 18:07:08.646876 1 redis_sentinel_k8s.go:398] New Master is rd-dev-1.rd-dev.default:6379, All the slaves are re-configured to replicate from this +I0412 18:07:08.647040 1 redis_sentinel_k8s.go:421] Redis-Sentinal-micro Finished +``` + +As you can see `rd-dev-1` is the new master while `rd-dev-0` and `rd-dev-2` are its slaves +Lets check if our original key `foo` is retained across master restarts +``` +$kubectl exec -i -t rd-dev-0 -c redis -- redis-cli -h rd-dev-0.rd-dev.default -p 6379 get foo +"bar" +``` + +#### Note: Care should be taken when you scale down the number of replicas of this statefulset. Please make sure that current redis-master is not among _to be scaled down_ replicas. diff --git a/incubator/redis-cache/templates/NOTES.txt b/incubator/redis-cache/templates/NOTES.txt new file mode 100644 index 0000000000..dc3302d301 --- /dev/null +++ b/incubator/redis-cache/templates/NOTES.txt @@ -0,0 +1,2 @@ +1. To Find which replica is the master + kubectl exec -i -t {{ template "fullname" . }}-0 -c redis -- redis-cli -h {{ template "fullname" . }}-0.{{ template "fullname" . }}.{{ .Release.Namespace }} -p 6379 info replication diff --git a/incubator/redis-cache/templates/_helpers.tpl b/incubator/redis-cache/templates/_helpers.tpl new file mode 100644 index 0000000000..f0d83d2edb --- /dev/null +++ b/incubator/redis-cache/templates/_helpers.tpl @@ -0,0 +1,16 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/incubator/redis-cache/templates/pdb.yaml b/incubator/redis-cache/templates/pdb.yaml new file mode 100644 index 0000000000..adb0fc7658 --- /dev/null +++ b/incubator/redis-cache/templates/pdb.yaml @@ -0,0 +1,12 @@ +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "fullname" . }} +spec: + selector: + matchLabels: + app: {{ template "name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} \ No newline at end of file diff --git a/incubator/redis-cache/templates/service.yaml b/incubator/redis-cache/templates/service.yaml new file mode 100644 index 0000000000..e4cbe47ec4 --- /dev/null +++ b/incubator/redis-cache/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + name: {{ template "fullname" . }} + labels: + app: {{ template "name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + ports: + - port: {{ .Values.service.port }} + name: {{ .Values.service.name }} + clusterIP: None + selector: + app: {{ template "name" . }} \ No newline at end of file diff --git a/incubator/redis-cache/templates/ss.yaml b/incubator/redis-cache/templates/ss.yaml new file mode 100644 index 0000000000..b9a4c85738 --- /dev/null +++ b/incubator/redis-cache/templates/ss.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: "{{ template "fullname" . }}" + labels: + app: {{ template "name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + serviceName: "{{ template "fullname" . }}" + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app: {{ template "name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + annotations: + pod.alpha.kubernetes.io/initialized: "true" + pod.beta.kubernetes.io/init-containers: '[ + { + "name":"sentinel-micro", + "image":"{{ .Values.microSentinel.image.repository }}:{{ .Values.microSentinel.image.tag }}", + "imagePullPolicy": "{{ .Values.microSentinel.image.pullPolicy }}", + "args" : ["-service","{{ template "fullname" . }}"], + "resources": +{{ toJson .Values.microSentinel.resources | indent 16 }}, + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef" : { + "apiVersion" : "v1", + "fieldPath" : "metadata.namespace" + } + } + } + ], + "volumeMounts": [ + { + "name": "config", + "mountPath": "/config" + } + ] + } + ]' + spec: + terminationGracePeriodSeconds: 5 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 5 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - "{{ template "name" . }}" + topologyKey: "kubernetes.io/hostname" + containers: + - name: redis + image: {{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }} + imagePullPolicy: {{ .Values.redis.image.pullPolicy }} + ports: + - containerPort: {{ .Values.service.port }} + name: {{ .Values.service.name }} + resources: +{{ toYaml .Values.redis.resources | indent 12 }} + - name: make-slave + image: {{ .Values.makeSlave.image.repository }}:{{ .Values.makeSlave.image.tag }} + imagePullPolicy: {{ .Values.makeSlave.image.pullPolicy }} + resources: +{{ toYaml .Values.makeSlave.resources | indent 12 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: "v1" + fieldPath: metadata.namespace + volumeMounts: + - name: config + mountPath: /config + volumes: + - name: config + emptyDir: {} \ No newline at end of file diff --git a/incubator/redis-cache/values.yaml b/incubator/redis-cache/values.yaml new file mode 100644 index 0000000000..77ed4ef1d3 --- /dev/null +++ b/incubator/redis-cache/values.yaml @@ -0,0 +1,49 @@ +# Default values for redis-cache. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +##Statefulset related configurations ss.yaml +replicaCount: 3 +redis: + image: + repository: redis + tag: 3.0-alpine + pullPolicy: IfNotPresent + resources: + limits: + memory: 128Mi + requests: + memory: 128Mi +microSentinel: + image: + repository: dhilipkumars/redis-sentinel-k8s + tag: 0.1.0 + pullPolicy: IfNotPresent + resources: {} + # limits: + # cpu: 10m + # memory: 20Mi + # requests: + # cpu: 10m + # memory: 20Mi +makeSlave: + image: + repository: dhilipkumars/mk-redis-slave + tag: 0.1.0 + pullPolicy: IfNotPresent + resources: {} + # limits: + # cpu: 10m + # memory: 20Mi + # requests: + # cpu: 10m + # memory: 20Mi + +# Service related configurations common for templates/service.yaml and templates/ss.yaml (redis container) +service: + name: rd-port + port: 6379 + + # Pod Disruption budget related configurations +podDisruptionBudget: + minAvailable: 2 \ No newline at end of file