From 5c8e924545a110b744a37e35d0c1ffd6857cc36e Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Sun, 12 Aug 2018 13:25:53 -0500 Subject: [PATCH] Make cockroachdb chart work with --wait flag on helm install (#6964) We had a deadlock of sorts. --wait waits until all pods are ready. postinstall hooks don't run until after --wait is done waiting. The pods won't be ready until the `init` job runs, which was a postinstall hook. This gets around that by making the init command run as part of the chart rather than as a postinstall hook. I added a bash retry loop for the init command to get around the lack of ability to configure pod restart backoffs in Kubernetes. --- stable/cockroachdb/Chart.yaml | 2 +- .../cockroachdb/templates/cluster-init.yaml | 22 ++++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/stable/cockroachdb/Chart.yaml b/stable/cockroachdb/Chart.yaml index fe5bf5138f..78c3601b65 100755 --- a/stable/cockroachdb/Chart.yaml +++ b/stable/cockroachdb/Chart.yaml @@ -1,6 +1,6 @@ name: cockroachdb home: https://www.cockroachlabs.com -version: 1.1.1 +version: 1.2.0 appVersion: 2.0.0 description: CockroachDB is a scalable, survivable, strongly-consistent SQL database. icon: https://raw.githubusercontent.com/cockroachdb/cockroach/master/docs/media/cockroach_db.png diff --git a/stable/cockroachdb/templates/cluster-init.yaml b/stable/cockroachdb/templates/cluster-init.yaml index e80d13261d..ce7680b0bd 100644 --- a/stable/cockroachdb/templates/cluster-init.yaml +++ b/stable/cockroachdb/templates/cluster-init.yaml @@ -6,11 +6,6 @@ metadata: heritage: {{.Release.Service | quote }} release: {{.Release.Name | quote }} chart: "{{.Chart.Name}}-{{.Chart.Version}}" - annotations: - # This is what defines this resource as a hook. Without this line, the - # job is considered part of the release. - "helm.sh/hook": post-install - "helm.sh/hook-delete-policy": hook-succeeded spec: template: spec: @@ -49,15 +44,16 @@ spec: - name: client-certs mountPath: /cockroach-certs {{- end }} + # Run the command in an `until` loop because this job is bound to come + # up before the cockroach pods (due to the time needed to get + # persistent volumes attached to nodes), and sleeping 5 seconds between + # attempts is much better than letting the pod fail when the init + # command does and waiting out Kubernetes' non-configurable exponential + # backoff for pod restarts. command: - - "/cockroach/cockroach" - - "init" -{{- if .Values.Secure.Enabled }} - - "--certs-dir=/cockroach-certs" -{{- else }} - - "--insecure" -{{- end }} - - "--host={{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}-0.{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + - "/bin/bash" + - "-ecx" + - "until /cockroach/cockroach init {{ if .Values.Secure.Enabled }}--certs-dir=/cockroach-certs{{ else }}--insecure{{ end }} --host={{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}-0.{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}; do sleep 5; done" restartPolicy: OnFailure {{- if .Values.Secure.Enabled }} volumes: