From 09892a393d04dc62bead7c3df66ec564159febb4 Mon Sep 17 00:00:00 2001 From: Michael Haselton Date: Wed, 5 Jul 2017 11:43:17 -0400 Subject: [PATCH] Harden elasticsearch chart for Kube 1.5 (#1062) * Update elasticsearch chart to work with Kube 1.5 * Add environment variable KUBERNETES_MASTER, resolves issue documented here: https://github.com/fabric8io/fabric8/issues/6229#issuecomment-247373967 * Rename PetSet to StatefulSet, rename template file * Add initialDelay and increase timesouts to all liveness and readiness checks. This was the only way I could get it to deploy reliably in my environment. * Update to a newer image version * Harden aspects of the elasticsearch chart * Added configmap to explicitly provide cluster configurations and scripts * Replace depreciating `ES_HEAP_SIZE` with `ES_JAVA_OPTS` to position for ES v5 support * Removed alpha storage class operators * Removed catastrophic liveness probe checking entire clusters health * Readiness probe now inspects local node health * Added termination grace period (defaults to 15m) to allow pre-stop-script.sh time to gracefully migrate shards * Added init container to configure `vm.max_map_count` * Updated elasticsearch.yaml: * Added `PROCESSOR` configuration to prevent large cluster garbage collection issues leading to node eviction * Added configurable gateway defaults to help avoid a split brain, requiring two masters online and in consensus before recovery can continue * Updated pre-stop-script.sh: * Check `v1beta1` `statefulset` endpoint * Evalute `.spec.replicas` for statefulset desired size * Clear `_cluster/settings` ip exclusion prior to shutdown to avoid a possible (random) ip match scenario on expansion of the clsuter * Data nodes now use default storage class if once is not specified * Apply best practices * Add Notes for client service types, and warnings --- incubator/elasticsearch/Chart.yaml | 6 +- incubator/elasticsearch/README.md | 69 +++++------ incubator/elasticsearch/templates/NOTES.txt | 31 +++++ .../elasticsearch/templates/_helpers.tpl | 43 +++++++ .../elasticsearch-client-deployment.yaml | 84 +++++++------ .../templates/elasticsearch-client-svc.yaml | 19 +++ .../templates/elasticsearch-cluster-svc.yaml | 16 --- .../templates/elasticsearch-configmap.yaml | 111 ++++++++++++++++++ .../templates/elasticsearch-data-petset.yaml | 82 ------------- .../elasticsearch-data-statefulset.yaml | 99 ++++++++++++++++ .../templates/elasticsearch-data-svc.yaml | 18 --- .../elasticsearch-master-deployment.yaml | 84 +++++++------ .../templates/elasticsearch-master-svc.yaml | 19 +++ .../elasticsearch-service-account.yaml | 11 +- .../templates/elasticsearch-svc.yaml | 15 --- incubator/elasticsearch/values.yaml | 75 +++++++----- 16 files changed, 510 insertions(+), 272 deletions(-) create mode 100644 incubator/elasticsearch/templates/NOTES.txt create mode 100644 incubator/elasticsearch/templates/_helpers.tpl create mode 100644 incubator/elasticsearch/templates/elasticsearch-client-svc.yaml delete mode 100644 incubator/elasticsearch/templates/elasticsearch-cluster-svc.yaml create mode 100644 incubator/elasticsearch/templates/elasticsearch-configmap.yaml delete mode 100644 incubator/elasticsearch/templates/elasticsearch-data-petset.yaml create mode 100644 incubator/elasticsearch/templates/elasticsearch-data-statefulset.yaml delete mode 100644 incubator/elasticsearch/templates/elasticsearch-data-svc.yaml create mode 100644 incubator/elasticsearch/templates/elasticsearch-master-svc.yaml delete mode 100644 incubator/elasticsearch/templates/elasticsearch-svc.yaml diff --git a/incubator/elasticsearch/Chart.yaml b/incubator/elasticsearch/Chart.yaml index 2e751ad08d..f561acd8c0 100755 --- a/incubator/elasticsearch/Chart.yaml +++ b/incubator/elasticsearch/Chart.yaml @@ -1,11 +1,15 @@ name: elasticsearch home: https://www.elastic.co/products/elasticsearch -version: 0.1.4 +version: 0.1.6 description: Flexible and powerful open source, distributed real-time search and analytics engine. icon: https://static-www.elastic.co/assets/blteb1c97719574938d/logo-elastic-elasticsearch-lt.svg sources: - https://www.elastic.co/products/elasticsearch - https://github.com/jetstack/elasticsearch-pet + - https://github.com/giantswarm/kubernetes-elastic-stack + - https://github.com/GoogleCloudPlatform/elasticsearch-docker maintainers: - name: Christian Simon email: christian@jetstack.io + - name: Michael Haselton + email: michael.haselton@gmail.com diff --git a/incubator/elasticsearch/README.md b/incubator/elasticsearch/README.md index f9af1b3b0d..ed4448a6b7 100644 --- a/incubator/elasticsearch/README.md +++ b/incubator/elasticsearch/README.md @@ -7,14 +7,14 @@ elasticsearch and their ## Prerequisites Details -* Kubernetes 1.3 with alpha APIs enabled +* Kubernetes 1.5 * PV dynamic provisioning support on the underlying infrastructure -## PetSet Details -* http://kubernetes.io/docs/user-guide/petset/ +## StatefulSets Details +* https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ -## PetSet Caveats -* http://kubernetes.io/docs/user-guide/petset/#alpha-limitations +## StatefulSets Caveats +* https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#limitations ## Todo @@ -25,9 +25,9 @@ elasticsearch and their ## Chart Details This chart will do the following: -* Implemented a dynamically scalable elasticsearch cluster using Kubernetes PetSets/Deployments +* Implemented a dynamically scalable elasticsearch cluster using Kubernetes StatefulSets/Deployments * Multi-role deployment: master, client and data nodes -* PetSet Supports scaling down without degrading the cluster +* Statefulset Supports scaling down without degrading the cluster ## Installing the Chart @@ -51,33 +51,27 @@ $ kubectl delete pvcs -l release=my-release,type=data The following tables lists the configurable parameters of the elasticsearch chart and their default values. -| Parameter | Description | Default | -|---------------------------|-----------------------------------|----------------------------------------------------------| -| `Image` | Container image name | `jetstack/elasticsearch-pet` | -| `ImageTag` | Container image tag | `2.3.4` | -| `ImagePullPolicy` | Container pull policy | `Always` | -| `ClientReplicas` | Client node replicas (deployment) | `2` | -| `ClientCpuRequests` | Client node requested cpu | `25m` | -| `ClientMemoryRequests` | Client node requested memory | `256Mi` | -| `ClientCpuLimits` | Client node requested cpu | `100m` | -| `ClientMemoryLimits` | Client node requested memory | `512Mi` | -| `ClientHeapSize` | Client node heap size | `128m` | -| `MasterReplicas` | Master node replicas (deployment) | `2` | -| `MasterCpuRequests` | Master node requested cpu | `25m` | -| `MasterMemoryRequests` | Master node requested memory | `256Mi` | -| `MasterCpuLimits` | Master node requested cpu | `100m` | -| `MasterMemoryLimits` | Master node requested memory | `512Mi` | -| `MasterHeapSize` | Master node heap size | `128m` | -| `DataReplicas` | Data node replicas (petset) | `3` | -| `DataCpuRequests` | Data node requested cpu | `250m` | -| `DataMemoryRequests` | Data node requested memory | `2Gi` | -| `DataCpuLimits` | Data node requested cpu | `1` | -| `DataMemoryLimits` | Data node requested memory | `4Gi` | -| `DataHeapSize` | Data node heap size | `1536m` | -| `DataStorage` | Data persistent volume size | `30Gi` | -| `DataStorageClass` | Data persistent volume Class | `anything` | -| `DataStorageClassVersion` | Version of StorageClass | `alpha` | -| `Component` | Selector Key | `elasticsearch` | +| Parameter | Description | Default | +| ------------------------------------ | --------------------------------------- | ----------------------------------- | +| `image.repository` | Container image name | `jetstack/elasticsearch-pet` | +| `image.tag` | Container image tag | `2.4.0` | +| `image.pullPolicy` | Container pull policy | `Always` | +| `client.name` | Client component name | `client` | +| `client.replicas` | Client node replicas (deployment) | `2` | +| `client.resources` | Client node resources requests & limits | `{} - cpu limit must be an integer` | +| `client.heapSize` | Client node heap size | `128m` | +| `client.serviceType` | Client service type | `ClusterIP` | +| `master.name` | Master component name | `master` | +| `master.replicas` | Master node replicas (deployment) | `2` | +| `master.resources` | Master node resources requests & limits | `{} - cpu limit must be an integer` | +| `master.heapSize` | Master node heap size | `128m` | +| `master.name` | Data component name | `data` | +| `data.replicas` | Data node replicas (statefulset) | `3` | +| `data.resources` | Data node resources requests & limits | `{} - cpu limit must be an integer` | +| `data.heapSize` | Data node heap size | `1536m` | +| `data.storage` | Data persistent volume size | `30Gi` | +| `data.storageClass` | Data persistent volume Class | `nil` | +| `data.terminationGracePeriodSeconds` | Data termination grace period (seconds) | `3600` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. @@ -102,7 +96,7 @@ would degrade performance heaviliy. The issue is tracked in ## Select right storage class for SSD volumes -### GCE + Kubernetes 1.4 +### GCE + Kubernetes 1.5 Create StorageClass for SSD-PD @@ -117,9 +111,8 @@ parameters: type: pd-ssd EOF ``` -Create cluster with Storage class `ssd` on Kubernetes 1.4+ +Create cluster with Storage class `ssd` on Kubernetes 1.5+ ``` -$ helm install incubator/elasticsearch --name my-release --set DataStorageClass=ssd,DataStorageClassVersion=beta - +$ helm install incubator/elasticsearch --name my-release --set data.storageClass=ssd,data.storage=100Gi ``` diff --git a/incubator/elasticsearch/templates/NOTES.txt b/incubator/elasticsearch/templates/NOTES.txt new file mode 100644 index 0000000000..de5071f85d --- /dev/null +++ b/incubator/elasticsearch/templates/NOTES.txt @@ -0,0 +1,31 @@ +The elasticsearch cluster has been installed. + +Elasticsearch can be accessed: + + * Within your cluster, at the following DNS name at port 9200: + + {{ template "client.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local + + * From outside the cluster, run these commands in the same shell: + {{- if contains "NodePort" .Values.client.serviceType }} + + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "client.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.client.serviceType }} + + WARNING: You have likely exposed your Elasticsearch cluster direct to the internet. + Elasticsearch does not implement any security for public facing clusters by default. + As a minimum level of security; switch to ClusterIP/NodePort and place an Nginx gateway infront of the cluster in order to lock down access to dangerous HTTP endpoints and verbs. + + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "client.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "client.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:9200 + {{- else if contains "ClusterIP" .Values.client.serviceType }} + + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "name" . }},component={{ .Values.client.name }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:9200 to use Elasticsearch" + kubectl port-forward --namespace {{ .Release.Namespace }} $POD_NAME 9200:9200 + {{- end }} diff --git a/incubator/elasticsearch/templates/_helpers.tpl b/incubator/elasticsearch/templates/_helpers.tpl new file mode 100644 index 0000000000..172629fa06 --- /dev/null +++ b/incubator/elasticsearch/templates/_helpers.tpl @@ -0,0 +1,43 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified client name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "client.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.client.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified data name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "data.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.data.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified master name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "master.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.master.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/incubator/elasticsearch/templates/elasticsearch-client-deployment.yaml b/incubator/elasticsearch/templates/elasticsearch-client-deployment.yaml index 7ed58990c9..3edc47f0bc 100644 --- a/incubator/elasticsearch/templates/elasticsearch-client-deployment.yaml +++ b/incubator/elasticsearch/templates/elasticsearch-client-deployment.yaml @@ -1,30 +1,44 @@ -apiVersion: extensions/v1beta1 +apiVersion: apps/v1beta1 kind: Deployment metadata: - name: "{{ printf "%s-client-%s" .Release.Name .Values.Name | trunc 24 }}" labels: - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - component: "{{.Release.Name}}-{{.Values.Component}}" - type: client + app: {{ template "name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + component: "{{ .Values.client.name }}" + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "client.fullname" . }} spec: - replicas: {{default 2 .Values.ClientReplicas }} + replicas: {{ .Values.client.replicas }} template: metadata: labels: - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - component: "{{.Release.Name}}-{{.Values.Component}}" - type: client + app: {{ template "name" . }} + component: "{{ .Values.client.name }}" + release: {{ .Release.Name }} + annotations: + # see https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html + # and https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration-memory.html#mlockall + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "sysctl", + "image": "busybox", + "imagePullPolicy": "Always", + "command": ["sysctl", "-w", "vm.max_map_count=262144"], + "securityContext": { + "privileged": true + } + } + ]' spec: - serviceAccountName: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" + serviceAccountName: {{ template "fullname" . }} containers: - name: elasticsearch env: - name: SERVICE - value: "{{ printf "%s-cluster-%s" .Release.Name .Values.Name | trunc 24 }}" + value: {{ template "master.fullname" . }} + - name: KUBERNETES_MASTER + value: kubernetes.default.svc.cluster.local - name: KUBERNETES_NAMESPACE valueFrom: fieldRef: @@ -33,30 +47,34 @@ spec: value: "false" - name: NODE_MASTER value: "false" - - name: ES_HEAP_SIZE - value: "{{.Values.ClientHeapSize}}" + - name: PROCESSORS + valueFrom: + resourceFieldRef: + resource: limits.cpu + - name: ES_JAVA_OPTS + value: "-Djava.net.preferIPv4Stack=true -Xms{{ .Values.client.heapSize }} -Xmx{{ .Values.client.heapSize }}" resources: - requests: - cpu: "{{.Values.ClientCpuRequests}}" - memory: "{{.Values.ClientMemoryRequests}}" - limits: - cpu: "{{.Values.ClientCpuLimits}}" - memory: "{{.Values.ClientMemoryLimits}}" - livenessProbe: - httpGet: - path: / - port: 9200 - initialDelaySeconds: 30 - timeoutSeconds: 1 +{{ toYaml .Values.client.resources | indent 12 }} readinessProbe: httpGet: - path: / + path: /_cluster/health?local=true port: 9200 - timeoutSeconds: 5 - image: "{{.Values.Image}}:{{.Values.ImageTag}}" - imagePullPolicy: "{{.Values.ImagePullPolicy}}" + initialDelaySeconds: 5 + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ default "" .Values.image.pullPolicy | quote }} ports: - containerPort: 9200 name: http - containerPort: 9300 name: transport + volumeMounts: + - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + name: config + subPath: elasticsearch.yml + - mountPath: /usr/share/elasticsearch/config/logging.yml + name: config + subPath: logging.yml + volumes: + - name: config + configMap: + name: {{ template "fullname" . }} diff --git a/incubator/elasticsearch/templates/elasticsearch-client-svc.yaml b/incubator/elasticsearch/templates/elasticsearch-client-svc.yaml new file mode 100644 index 0000000000..32f2095efa --- /dev/null +++ b/incubator/elasticsearch/templates/elasticsearch-client-svc.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: {{ template "name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + component: "{{ .Values.client.name }}" + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "client.fullname" . }} +spec: + ports: + - port: 9200 + targetPort: http + selector: + app: {{ template "name" . }} + component: "{{ .Values.client.name }}" + release: {{ .Release.Name }} + type: {{ .Values.client.serviceType }} diff --git a/incubator/elasticsearch/templates/elasticsearch-cluster-svc.yaml b/incubator/elasticsearch/templates/elasticsearch-cluster-svc.yaml deleted file mode 100644 index afb82c9640..0000000000 --- a/incubator/elasticsearch/templates/elasticsearch-cluster-svc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: "{{ printf "%s-cluster-%s" .Release.Name .Values.Name | trunc 24 }}" - labels: - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" -spec: - clusterIP: None - ports: - - port: 9300 - targetPort: 9300 - selector: - component: "{{.Release.Name}}-{{.Values.Component}}" - diff --git a/incubator/elasticsearch/templates/elasticsearch-configmap.yaml b/incubator/elasticsearch/templates/elasticsearch-configmap.yaml new file mode 100644 index 0000000000..8a28611e7f --- /dev/null +++ b/incubator/elasticsearch/templates/elasticsearch-configmap.yaml @@ -0,0 +1,111 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "fullname" . }} + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +data: + elasticsearch.yml: |- + node.data: ${NODE_DATA:true} + node.master: ${NODE_MASTER:true} + node.name: ${HOSTNAME} + + # see https://github.com/kubernetes/kubernetes/issues/3595 + bootstrap.mlockall: ${BOOTSTRAP_MLOCKALL:false} + + network.host: 0.0.0.0 + + cloud: + kubernetes: + service: ${SERVICE} + namespace: ${KUBERNETES_NAMESPACE} + + discovery: + type: kubernetes + zen: + minimum_master_nodes: 2 + + # see https://github.com/elastic/elasticsearch-definitive-guide/pull/679 + processors: ${PROCESSORS:} + + # avoid split-brain w/ a minimum consensus of two masters plus a data node + gateway.expected_master_nodes: ${EXPECTED_MASTER_NODES:2} + gateway.expected_data_nodes: ${EXPECTED_DATA_NODES:1} + gateway.recover_after_time: ${RECOVER_AFTER_TIME:5m} + gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:2} + gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:1} + logging.yml: |- + # you can override this using by setting a system property, for example -Des.logger.level=DEBUG + es.logger.level: INFO + rootLogger: ${es.logger.level}, console + logger: + # log action execution errors for easier debugging + action: DEBUG + # reduce the logging for aws, too much is logged under the default INFO + com.amazonaws: WARN + + appender: + console: + type: console + layout: + type: consolePattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" + pre-stop-hook.sh: |- + #!/bin/bash + set -e + + SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount + KUBE_TOKEN=$(<${SERVICE_ACCOUNT_PATH}/token) + KUBE_NAMESPACE=$(<${SERVICE_ACCOUNT_PATH}/namespace) + + STATEFULSET_NAME=$(echo "${HOSTNAME}" | sed 's/-[0-9]*$//g') + INSTANCE_ID=$(echo "${HOSTNAME}" | grep -o '[0-9]*$') + + echo "Prepare stopping of Pet ${KUBE_NAMESPACE}/${HOSTNAME} of StatefulSet ${KUBE_NAMESPACE}/${STATEFULSET_NAME} instance_id ${INSTANCE_ID}" + + INSTANCES_DESIRED=$(curl -s \ + --cacert ${SERVICE_ACCOUNT_PATH}/ca.crt \ + -H "Authorization: Bearer $KUBE_TOKEN" \ + "https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_PORT_443_TCP_PORT}/apis/apps/v1beta1/namespaces/${KUBE_NAMESPACE}/statefulsets/${STATEFULSET_NAME}/status" | jq -r '.spec.replicas') + + echo "Desired instance count is ${INSTANCES_DESIRED}" + + if [ "${INSTANCE_ID}" -lt "${INSTANCES_DESIRED}" ]; then + echo "No data migration needed" + exit 0 + fi + + echo "Prepare to migrate data of the node" + + NODE_STATS=$(curl -s -XGET 'http://localhost:9200/_nodes/stats') + NODE_IP=$(echo "${NODE_STATS}" | jq -r ".nodes[] | select(.name==\"${HOSTNAME}\") | .host") + + echo "Move all data from node ${NODE_IP}" + + curl -s -XPUT localhost:9200/_cluster/settings -d "{ + \"transient\" :{ + \"cluster.routing.allocation.exclude._ip\" : \"${NODE_IP}\" + } + }" + echo + + echo "Wait for node to become empty" + DOC_COUNT=$(echo "${NODE_STATS}" | jq ".nodes[] | select(.name==\"${HOSTNAME}\") | .indices.docs.count") + while [ "${DOC_COUNT}" -gt 0 ]; do + NODE_STATS=$(curl -s -XGET 'http://localhost:9200/_nodes/stats') + DOC_COUNT=$(echo "${NODE_STATS}" | jq -r ".nodes[] | select(.name==\"${HOSTNAME}\") | .indices.docs.count") + echo "Node contains ${DOC_COUNT} documents" + sleep 1 + done + + curl -s -XPUT localhost:9200/_cluster/settings -d "{ + \"transient\" :{ + \"cluster.routing.allocation.exclude._ip\" : \"\" + } + }" + echo + + echo "Node clear to shutdown" diff --git a/incubator/elasticsearch/templates/elasticsearch-data-petset.yaml b/incubator/elasticsearch/templates/elasticsearch-data-petset.yaml deleted file mode 100644 index cd4746c418..0000000000 --- a/incubator/elasticsearch/templates/elasticsearch-data-petset.yaml +++ /dev/null @@ -1,82 +0,0 @@ -apiVersion: apps/v1alpha1 -kind: PetSet -metadata: - name: "{{ printf "%s-data-%s" .Release.Name .Values.Name | trunc 24 }}" - labels: - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - component: "{{.Release.Name}}-{{.Values.Component}}" - type: data -spec: - serviceName: "{{ printf "%s-data-%s" .Release.Name .Values.Name | trunc 24 }}" - replicas: {{default 3 .Values.DataReplicas }} - template: - metadata: - labels: - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - component: "{{.Release.Name}}-{{.Values.Component}}" - type: data - annotations: - pod.alpha.kubernetes.io/initialized: "true" - spec: - serviceAccountName: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" - containers: - - name: elasticsearch - env: - - name: SERVICE - value: "{{ printf "%s-cluster-%s" .Release.Name .Values.Name | trunc 24 }}" - - name: KUBERNETES_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: NODE_MASTER - value: "false" - - name: ES_HEAP_SIZE - value: "{{.Values.DataHeapSize}}" - image: "{{.Values.Image}}:{{.Values.ImageTag}}" - imagePullPolicy: "{{.Values.ImagePullPolicy}}" - ports: - - containerPort: 9300 - name: transport - resources: - requests: - cpu: "{{.Values.DataCpuRequests}}" - memory: "{{.Values.DataMemoryRequests}}" - limits: - cpu: "{{.Values.DataCpuLimits}}" - memory: "{{.Values.DataMemoryLimits}}" - livenessProbe: - httpGet: - path: / - port: 9200 - initialDelaySeconds: 30 - timeoutSeconds: 1 - readinessProbe: - httpGet: - path: / - port: 9200 - timeoutSeconds: 5 - volumeMounts: - - mountPath: /usr/share/elasticsearch/data - name: elasticsearch-data - lifecycle: - preStop: - exec: - command: ["/bin/bash","/pre-stop-hook.sh"] - volumeClaimTemplates: - - metadata: - name: elasticsearch-data - annotations: - volume.{{ .Values.DataStorageClassVersion }}.kubernetes.io/storage-class: "{{ .Values.DataStorageClass }}" - spec: - accessModes: [ ReadWriteOnce ] - resources: - requests: - storage: "{{.Values.DataStorage}}" diff --git a/incubator/elasticsearch/templates/elasticsearch-data-statefulset.yaml b/incubator/elasticsearch/templates/elasticsearch-data-statefulset.yaml new file mode 100644 index 0000000000..30aaba2c13 --- /dev/null +++ b/incubator/elasticsearch/templates/elasticsearch-data-statefulset.yaml @@ -0,0 +1,99 @@ +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + labels: + app: {{ template "name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + component: "{{ .Values.data.name }}" + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "data.fullname" . }} +spec: + serviceName: {{ template "data.fullname" . }} + replicas: {{ .Values.data.replicas }} + template: + metadata: + labels: + app: {{ template "name" . }} + component: "{{ .Values.data.name }}" + release: {{ .Release.Name }} + annotations: + pod.alpha.kubernetes.io/initialized: "true" + # see https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html + # and https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration-memory.html#mlockall + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "sysctl", + "image": "busybox", + "imagePullPolicy": "Always", + "command": ["sysctl", "-w", "vm.max_map_count=262144"], + "securityContext": { + "privileged": true + } + } + ]' + spec: + serviceAccountName: {{ template "fullname" . }} + containers: + - name: elasticsearch + env: + - name: SERVICE + value: {{ template "master.fullname" . }} + - name: KUBERNETES_MASTER + value: kubernetes.default.svc.cluster.local + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_MASTER + value: "false" + - name: PROCESSORS + valueFrom: + resourceFieldRef: + resource: limits.cpu + - name: ES_JAVA_OPTS + value: "-Djava.net.preferIPv4Stack=true -Xms{{ .Values.data.heapSize }} -Xmx{{ .Values.data.heapSize }}" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ default "" .Values.image.pullPolicy | quote }} + ports: + - containerPort: 9300 + name: transport + resources: +{{ toYaml .Values.data.resources | indent 12 }} + readinessProbe: + httpGet: + path: /_cluster/health?local=true + port: 9200 + initialDelaySeconds: 5 + volumeMounts: + - mountPath: /usr/share/elasticsearch/data + name: data + - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + name: config + subPath: elasticsearch.yml + - mountPath: /usr/share/elasticsearch/config/logging.yml + name: config + subPath: logging.yml + - name: config + mountPath: /pre-stop-hook.sh + subPath: pre-stop-hook.sh + lifecycle: + preStop: + exec: + command: ["/bin/bash","/pre-stop-hook.sh"] + terminationGracePeriodSeconds: {{ .Values.data.terminationGracePeriodSeconds }} + volumes: + - name: config + configMap: + name: {{ template "fullname" . }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ ReadWriteOnce ] + {{- if .Values.data.storageClass }} + storageClassName: "{{ .Values.data.storageClass }}" + {{- end }} + resources: + requests: + storage: "{{ .Values.data.storage }}" diff --git a/incubator/elasticsearch/templates/elasticsearch-data-svc.yaml b/incubator/elasticsearch/templates/elasticsearch-data-svc.yaml deleted file mode 100644 index 47507f25e3..0000000000 --- a/incubator/elasticsearch/templates/elasticsearch-data-svc.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" - name: "{{ printf "%s-data-%s" .Release.Name .Values.Name | trunc 24 }}" - labels: - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" -spec: - clusterIP: None - ports: - - port: 9300 - targetPort: 9300 - selector: - component: "{{.Release.Name}}-{{.Values.Component}}" - type: data diff --git a/incubator/elasticsearch/templates/elasticsearch-master-deployment.yaml b/incubator/elasticsearch/templates/elasticsearch-master-deployment.yaml index e342b54427..663325a287 100644 --- a/incubator/elasticsearch/templates/elasticsearch-master-deployment.yaml +++ b/incubator/elasticsearch/templates/elasticsearch-master-deployment.yaml @@ -1,58 +1,76 @@ -apiVersion: extensions/v1beta1 +apiVersion: apps/v1beta1 kind: Deployment metadata: - name: "{{ printf "%s-master-%s" .Release.Name .Values.Name | trunc 24 }}" labels: - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - component: "{{.Release.Name}}-{{.Values.Component}}" - type: master + app: {{ template "name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + component: "{{ .Values.master.name }}" + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "master.fullname" . }} spec: - replicas: {{default 2 .Values.MasterReplicas }} + replicas: {{ .Values.master.replicas }} template: metadata: labels: - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - component: "{{.Release.Name}}-{{.Values.Component}}" - type: master + app: {{ template "name" . }} + component: "{{ .Values.master.name }}" + release: {{ .Release.Name }} + annotations: + # see https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html + # and https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration-memory.html#mlockall + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "sysctl", + "image": "busybox", + "imagePullPolicy": "Always", + "command": ["sysctl", "-w", "vm.max_map_count=262144"], + "securityContext": { + "privileged": true + } + } + ]' spec: - serviceAccountName: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" + serviceAccountName: {{ template "fullname" . }} containers: - name: elasticsearch env: - name: SERVICE - value: "{{ printf "%s-cluster-%s" .Release.Name .Values.Name | trunc 24 }}" + value: {{ template "master.fullname" . }} + - name: KUBERNETES_MASTER + value: kubernetes.default.svc.cluster.local - name: KUBERNETES_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: NODE_DATA value: "false" - - name: ES_HEAP_SIZE - value: "{{.Values.MasterHeapSize}}" + - name: PROCESSORS + valueFrom: + resourceFieldRef: + resource: limits.cpu + - name: ES_JAVA_OPTS + value: "-Djava.net.preferIPv4Stack=true -Xms{{ .Values.master.heapSize }} -Xmx{{ .Values.master.heapSize }}" resources: - requests: - cpu: "{{.Values.MasterCpuRequests}}" - memory: "{{.Values.MasterMemoryRequests}}" - limits: - cpu: "{{.Values.MasterCpuLimits}}" - memory: "{{.Values.MasterMemoryLimits}}" - livenessProbe: - httpGet: - path: / - port: 9200 - initialDelaySeconds: 30 - timeoutSeconds: 1 +{{ toYaml .Values.master.resources | indent 12 }} readinessProbe: httpGet: - path: / + path: /_cluster/health?local=true port: 9200 - timeoutSeconds: 5 - image: "{{.Values.Image}}:{{.Values.ImageTag}}" - imagePullPolicy: "{{.Values.ImagePullPolicy}}" + initialDelaySeconds: 5 + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ default "" .Values.image.pullPolicy | quote }} ports: - containerPort: 9300 name: transport + volumeMounts: + - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + name: config + subPath: elasticsearch.yml + - mountPath: /usr/share/elasticsearch/config/logging.yml + name: config + subPath: logging.yml + volumes: + - name: config + configMap: + name: {{ template "fullname" . }} diff --git a/incubator/elasticsearch/templates/elasticsearch-master-svc.yaml b/incubator/elasticsearch/templates/elasticsearch-master-svc.yaml new file mode 100644 index 0000000000..7d3f484231 --- /dev/null +++ b/incubator/elasticsearch/templates/elasticsearch-master-svc.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: {{ template "name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + component: "{{ .Values.master.name }}" + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "master.fullname" . }} +spec: + clusterIP: None + ports: + - port: 9300 + targetPort: 9300 + selector: + app: {{ template "name" . }} + component: "{{ .Values.master.name }}" + release: {{ .Release.Name }} diff --git a/incubator/elasticsearch/templates/elasticsearch-service-account.yaml b/incubator/elasticsearch/templates/elasticsearch-service-account.yaml index 13a3869935..b187517fe8 100644 --- a/incubator/elasticsearch/templates/elasticsearch-service-account.yaml +++ b/incubator/elasticsearch/templates/elasticsearch-service-account.yaml @@ -1,10 +1,9 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" labels: - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - component: "{{.Release.Name}}-{{.Values.Component}}" - type: master + app: {{ template "name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "fullname" . }} diff --git a/incubator/elasticsearch/templates/elasticsearch-svc.yaml b/incubator/elasticsearch/templates/elasticsearch-svc.yaml deleted file mode 100644 index ff176da154..0000000000 --- a/incubator/elasticsearch/templates/elasticsearch-svc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 24 }}" - labels: - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" -spec: - ports: - - port: 9200 - targetPort: http - selector: - component: "{{.Release.Name}}-{{.Values.Component}}" - type: client diff --git a/incubator/elasticsearch/values.yaml b/incubator/elasticsearch/values.yaml index 9567ff029b..1fda84a7f2 100644 --- a/incubator/elasticsearch/values.yaml +++ b/incubator/elasticsearch/values.yaml @@ -1,33 +1,48 @@ # Default values for elasticsearch. # This is a YAML-formatted file. -# Declare name/value pairs to be passed into your templates. -# name: value +# Declare variables to be passed into your templates. -Name: es -PeerPort: 2380 -ClientPort: 2379 -Component: "elasticsearch" -Image: "jetstack/elasticsearch-pet" -ImageTag: "2.3.5" -ImagePullPolicy: "Always" -ClientReplicas: 2 -ClientCpuRequests: "25m" -ClientMemoryRequests: "256Mi" -ClientCpuLimits: "100m" -ClientMemoryLimits: "512Mi" -ClientHeapSize: "128m" -MasterReplicas: 2 -MasterCpuRequests: "25m" -MasterMemoryRequests: "256Mi" -MasterCpuLimits: "100m" -MasterMemoryLimits: "512Mi" -MasterHeapSize: "128m" -DataReplicas: 3 -DataCpuRequests: "250m" -DataMemoryRequests: "2Gi" -DataCpuLimits: "1" -DataMemoryLimits: "4Gi" -DataHeapSize: "1536m" -DataStorage: "30Gi" -DataStorageClass: "anything" -DataStorageClassVersion: "alpha" +image: + repository: "jetstack/elasticsearch-pet" + tag: "2.4.0" + pullPolicy: "Always" + +client: + name: client + replicas: 2 + serviceType: ClusterIP + heapSize: "128m" + resources: + limits: + cpu: "1" + memory: "512Mi" + requests: + cpu: "25m" + memory: "256Mi" + +master: + name: master + replicas: 2 + heapSize: "128m" + resources: + limits: + cpu: "1" + memory: "512Mi" + requests: + cpu: "25m" + memory: "256Mi" + +data: + name: data + replicas: 3 + heapSize: "1536m" + storage: "30Gi" + # storageClass: "ssd" + terminationGracePeriodSeconds: 3600 + resources: + limits: + cpu: "1" + memory: "512Mi" + requests: + cpu: "25m" + memory: "256Mi"