diff --git a/stable/cockroachdb/Chart.yaml b/stable/cockroachdb/Chart.yaml index cb9846267c..ac89c68774 100755 --- a/stable/cockroachdb/Chart.yaml +++ b/stable/cockroachdb/Chart.yaml @@ -1,7 +1,7 @@ name: cockroachdb home: https://www.cockroachlabs.com -version: 0.6.7 -appVersion: 1.1.5 +version: 1.0.0 +appVersion: 1.1.7 description: CockroachDB is a scalable, survivable, strongly-consistent SQL database. icon: https://raw.githubusercontent.com/cockroachdb/cockroach/master/docs/media/cockroach_db.png sources: diff --git a/stable/cockroachdb/README.md b/stable/cockroachdb/README.md index dd94f68351..c45c2d2ce3 100644 --- a/stable/cockroachdb/README.md +++ b/stable/cockroachdb/README.md @@ -4,6 +4,10 @@ * Kubernetes 1.7 (for PodDisruptionBudget `MaxUnavailable` support -- you can run at Kubernetes 1.5 if you don't care about the PodDisruptionBudget) * PV support on the underlying infrastructure +* If you want to secure your cluster to use TLS certificates for all network + communication, [Helm must be installed with RBAC + privileges](https://github.com/kubernetes/helm/blob/master/docs/rbac.md) + or else you will get an "attempt to grant extra privileges" error. ## StatefulSet Details * http://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/ @@ -11,10 +15,8 @@ ## StatefulSet Caveats * http://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/#limitations -## Todo -* Support setting up clusters with certificate-based authentication - ## Chart Details + This chart will do the following: * Set up a dynamically scalable CockroachDB cluster using a Kubernetes StatefulSet @@ -27,6 +29,15 @@ To install the chart with the release name `my-release`: helm install --name my-release stable/cockroachdb ``` +If you are running in secure mode (with configuration parameter `Secure.Enabled` +set to `true`), then you will have to manually approve the cluster's security +certificates as the pods are created. You can see the pending +certificate-signing requests by running `kubectl get csr`, and approve them by +running `kubectl certificate approve `. You'll have to approve one +certificate for each node (e.g. `default.node.eerie-horse-cockroachdb-0` and +one client certificate for the job that initializes the cluster (e.g. +`default.node.root`). + ## Configuration The following table lists the configurable parameters of the CockroachDB chart and their default values. @@ -52,6 +63,11 @@ The following table lists the configurable parameters of the CockroachDB chart a | `NetworkPolicy.Enabled` | Enable NetworkPolicy | `false` | | `NetworkPolicy.AllowExternal` | Don't require client label for connections | `true` | | `Service.Type` | Public service type | `ClusterIP` | +| `Secure.Enabled` | Whether to run securely using TLS certificates | `false` | +| `Secure.RequestCertsImage` | Image to use for requesting TLS certificates | `cockroachdb/cockroach-k8s-request-cert` | +| `Secure.RequestCertsImageTag` | Image tag to use for requesting TLS certificates | `0.3` | +| `Secure.ServiceAccount.Create` | Whether to create a new RBAC service account | `true` | +| `Secure.ServiceAccount.Name` | Name of RBAC service account to use | `` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. @@ -113,6 +129,14 @@ Waiting for pod default/cockroach-client to terminate, status is Running pod "cockroach-client" deleted ``` +If you are running in secure mode, you will have to provide a client certificate +to the cluster in order to authenticate, so the above command will not work. See +[here](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/client-secure.yaml) +for an example of how to set up an interactive SQL shell against a secure +cluster or +[here](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/example-app-secure.yaml) +for an example application connecting to a secure cluster. + ## Cluster health Because our pod spec includes regular health checks of the CockroachDB processes, @@ -209,3 +233,7 @@ as shown below: ```shell kubectl scale statefulset my-release-cockroachdb --replicas=4 ``` + +Note that if you are running in secure mode and increase the size of your +cluster, you will also have to approve the certificate-signing request of each +new node (using `kubectl get csr` and `kubectl certificate approve`). diff --git a/stable/cockroachdb/templates/NOTES.txt b/stable/cockroachdb/templates/NOTES.txt index 5914cc86ab..6dc2c56f93 100644 --- a/stable/cockroachdb/templates/NOTES.txt +++ b/stable/cockroachdb/templates/NOTES.txt @@ -5,6 +5,7 @@ value to) at the following DNS name from within your cluster: Because CockroachDB supports the PostgreSQL wire protocol, you can connect to the cluster using any available PostgreSQL client. +{{- if not .Values.Secure.Enabled }} For example, you can open up a SQL shell to the cluster by running: kubectl run -it --rm cockroach-client \ @@ -13,22 +14,29 @@ For example, you can open up a SQL shell to the cluster by running: --labels="{{.Release.Name}}-{{.Values.Component}}-client=true" \{{- end }} --command -- ./cockroach sql --insecure --host {{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}-public.{{ .Release.Namespace }} +From there, you can interact with the SQL shell as you would any other SQL shell, +confident that any data you write will be safe and available even if parts of +your cluster fail. +{{- else }} +Note that because the cluster is running in secure mode, any client application that you attempt to connect will either need to have a valid client certificate or a valid username and password. +{{- end }} + {{ if and (.Values.NetworkPolicy.Enabled) (not .Values.NetworkPolicy.AllowExternal) }} Note: Since NetworkPolicy is enabled, only pods with label {{.Release.Name}}-{{.Values.Component}}-client=true" will be able to connect to this cockroachdb cluster. {{- end }} -From there, you can interact with the SQL shell as you would any other SQL shell, -confident that any data you write will be safe and available even if parts of -your cluster fail. - Finally, to open up the CockroachDB admin UI, you can port-forward from your local machine into one of the instances in the cluster: kubectl port-forward {{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}-0 8080 +{{- if not .Values.Secure.Enabled }} Then you can access the admin UI at http://localhost:8080/ in your web browser. +{{- else }} +Then you can access the admin UI at https://localhost:8080/ in your web browser. +{{- end }} For more information on using CockroachDB, please see the project's docs at https://www.cockroachlabs.com/docs/ diff --git a/stable/cockroachdb/templates/_helpers.tpl b/stable/cockroachdb/templates/_helpers.tpl index 0feb1ca12b..1aa6bbc9b5 100644 --- a/stable/cockroachdb/templates/_helpers.tpl +++ b/stable/cockroachdb/templates/_helpers.tpl @@ -8,3 +8,14 @@ Return the appropriate apiVersion for networkpolicy. {{- print "networking.k8s.io/v1" -}} {{- end -}} {{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cockroachdb.serviceAccountName" -}} +{{- if .Values.Secure.ServiceAccount.Create -}} + {{ default (printf "%s-%s" .Release.Name .Values.Name | trunc 56) .Values.Secure.ServiceAccount.Name }} +{{- else -}} + {{ default "default" .Values.Secure.ServiceAccount.Name }} +{{- end -}} +{{- end -}} diff --git a/stable/cockroachdb/templates/cluster-init.yaml b/stable/cockroachdb/templates/cluster-init.yaml index a7d1f32587..e80d13261d 100644 --- a/stable/cockroachdb/templates/cluster-init.yaml +++ b/stable/cockroachdb/templates/cluster-init.yaml @@ -14,13 +14,53 @@ metadata: spec: template: spec: +{{- if .Values.Secure.Enabled }} + serviceAccountName: {{ template "cockroachdb.serviceAccountName" . }} + initContainers: + # The init-certs container sends a certificate signing request to the + # kubernetes cluster. + # You can see pending requests using: kubectl get csr + # CSRs can be approved using: kubectl certificate approve + # + # In addition to the client certificate and key, the init-certs entrypoint will symlink + # the cluster CA to the certs directory. + - name: init-certs + image: "{{ .Values.Secure.RequestCertsImage }}:{{ .Values.Secure.RequestCertsImageTag }}" + imagePullPolicy: "{{ .Values.ImagePullPolicy }}" + command: + - "/bin/ash" + - "-ecx" + - "/request-cert -namespace=${POD_NAMESPACE} -certs-dir=/cockroach-certs -type=client -user=root -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: client-certs + mountPath: /cockroach-certs +{{- end }} containers: - name: cluster-init image: "{{ .Values.Image }}:{{ .Values.ImageTag }}" imagePullPolicy: "{{ .Values.ImagePullPolicy }}" +{{- if .Values.Secure.Enabled }} + volumeMounts: + - name: client-certs + mountPath: /cockroach-certs +{{- end }} command: - "/cockroach/cockroach" - "init" +{{- if .Values.Secure.Enabled }} + - "--certs-dir=/cockroach-certs" +{{- else }} - "--insecure" +{{- end }} - "--host={{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}-0.{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" restartPolicy: OnFailure +{{- if .Values.Secure.Enabled }} + volumes: + - name: client-certs + emptyDir: {} +{{- end }} diff --git a/stable/cockroachdb/templates/cockroachdb-statefulset.yaml b/stable/cockroachdb/templates/cockroachdb-statefulset.yaml index 1bac88b8a1..d09c5850fd 100644 --- a/stable/cockroachdb/templates/cockroachdb-statefulset.yaml +++ b/stable/cockroachdb/templates/cockroachdb-statefulset.yaml @@ -1,3 +1,90 @@ +{{- if .Values.Secure.Enabled }} +{{- if .Values.Secure.ServiceAccount.Create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "cockroachdb.serviceAccountName" . }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + component: "{{ .Release.Name }}-{{ .Values.Component }}" +--- +{{- end }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + component: "{{ .Release.Name }}-{{ .Values.Component }}" +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - get +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + component: "{{ .Release.Name }}-{{ .Values.Component }}" +rules: +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - create + - get + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + component: "{{ .Release.Name }}-{{ .Values.Component }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" +subjects: +- kind: ServiceAccount + name: {{ template "cockroachdb.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + component: "{{ .Release.Name }}-{{ .Values.Component }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" +subjects: +- kind: ServiceAccount + name: {{ template "cockroachdb.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +--- +{{- end }} apiVersion: v1 kind: Service metadata: @@ -85,6 +172,34 @@ spec: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" component: "{{ .Release.Name }}-{{ .Values.Component }}" spec: +{{- if .Values.Secure.Enabled }} + serviceAccountName: {{ template "cockroachdb.serviceAccountName" . }} + initContainers: + # The init-certs container sends a certificate signing request to the + # kubernetes cluster. + # You can see pending requests using: kubectl get csr + # CSRs can be approved using: kubectl certificate approve + # + # All addresses used to contact a node must be specified in the --addresses arg. + # + # In addition to the node certificate and key, the init-certs entrypoint will symlink + # the cluster CA to the certs directory. + - name: init-certs + image: "{{ .Values.Secure.RequestCertsImage }}:{{ .Values.Secure.RequestCertsImageTag }}" + imagePullPolicy: "{{ .Values.ImagePullPolicy }}" + command: + - "/bin/ash" + - "-ecx" + - "/request-cert -namespace=${POD_NAMESPACE} -certs-dir=/cockroach-certs -type=node -addresses=localhost,127.0.0.1,$(hostname -f),$(hostname -f|cut -f 1-2 -d '.'),{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}-public,{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}-public.$(hostname -f|cut -f 3- -d '.') -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: certs + mountPath: /cockroach-certs +{{- end }} affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -92,10 +207,10 @@ spec: podAffinityTerm: labelSelector: matchExpressions: - - key: app + - key: component operator: In values: - - cockroachdb + - "{{ .Release.Name }}-{{ .Values.Component }}" topologyKey: kubernetes.io/hostname containers: - name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" @@ -111,18 +226,32 @@ spec: env: - name: STATEFULSET_NAME value: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 56 }}" + - name: COCKROACH_CHANNEL + value: kubernetes-helm volumeMounts: - name: datadir mountPath: /cockroach/cockroach-data +{{- if .Values.Secure.Enabled }} + - name: certs + mountPath: /cockroach/cockroach-certs +{{- end }} command: - "/bin/bash" - "-ecx" # The use of qualified `hostname -f` is crucial: # Other nodes aren't able to look up the unqualified hostname. - - "exec /cockroach/cockroach start --logtostderr --insecure --advertise-host $(hostname -f) --http-host 0.0.0.0 --cache {{ .Values.CacheSize }} --max-sql-memory {{ .Values.MaxSQLMemory }} --join ${STATEFULSET_NAME}-0.${STATEFULSET_NAME},${STATEFULSET_NAME}-1.${STATEFULSET_NAME},${STATEFULSET_NAME}-2.${STATEFULSET_NAME}" + - "exec /cockroach/cockroach start --logtostderr {{ if .Values.Secure.Enabled }}--certs-dir /cockroach/cockroach-certs{{ else }}--insecure{{ end }} --advertise-host $(hostname -f) --http-host 0.0.0.0 --cache {{ .Values.CacheSize }} --max-sql-memory {{ .Values.MaxSQLMemory }} --join ${STATEFULSET_NAME}-0.${STATEFULSET_NAME},${STATEFULSET_NAME}-1.${STATEFULSET_NAME},${STATEFULSET_NAME}-2.${STATEFULSET_NAME}" # No pre-stop hook is required, a SIGTERM plus some time is all that's # needed for graceful shutdown of a node. terminationGracePeriodSeconds: 60 + volumes: + - name: datadir + persistentVolumeClaim: + claimName: datadir +{{- if .Values.Secure.Enabled }} + - name: certs + emptyDir: {} +{{- end }} volumeClaimTemplates: - metadata: name: datadir diff --git a/stable/cockroachdb/values.yaml b/stable/cockroachdb/values.yaml index cf4f443e7d..212a789072 100644 --- a/stable/cockroachdb/values.yaml +++ b/stable/cockroachdb/values.yaml @@ -33,3 +33,13 @@ NetworkPolicy: AllowExternal: true Service: type: ClusterIP +Secure: + Enabled: false + RequestCertsImage: "cockroachdb/cockroach-k8s-request-cert" + RequestCertsImageTag: "0.3" + ServiceAccount: + # Specifies whether a service account should be created. + Create: true + # The name of the service account to use. + # If not set and create is true, a name is generated. + Name: