From 8f7c90ce19cef7792b8280974bd6b814767b0ead Mon Sep 17 00:00:00 2001 From: Tim Stoop Date: Thu, 3 May 2018 19:43:51 +0200 Subject: [PATCH] [stable/consul] Provide advanced configuration options for consul (resubmit) (#4765) * added extra configuration options for consul * fixed readme * added pitfall for ACLs * added a note about configuration updates * updated documentation syntax and added example --- stable/consul/Chart.yaml | 2 +- stable/consul/README.md | 24 +++++++++++++++++++ .../consul/templates/consul-statefulset.yaml | 17 +++++++++++++ stable/consul/values.yaml | 6 +++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/stable/consul/Chart.yaml b/stable/consul/Chart.yaml index 1612940319..7e29cb1272 100755 --- a/stable/consul/Chart.yaml +++ b/stable/consul/Chart.yaml @@ -1,6 +1,6 @@ name: consul home: https://github.com/hashicorp/consul -version: 1.3.5 +version: 1.3.6 appVersion: 1.0.0 description: Highly available and distributed service discovery and key-value store designed with support for the modern data center to make distributed systems and diff --git a/stable/consul/README.md b/stable/consul/README.md index b3d47395e3..0c116f2c6d 100644 --- a/stable/consul/README.md +++ b/stable/consul/README.md @@ -32,6 +32,8 @@ The following table lists the configurable parameters of the consul chart and th | `ImagePullPolicy` | Container pull policy | `Always` | | `Replicas` | k8s statefulset replicas | `3` | | `Component` | k8s selector key | `consul` | +| `ConsulConfig` | List of secrets and configMaps containing consul configuration | [] | +| `Cpu` | container requested cpu | `100m` | | `DatacenterName` | Consul Datacenter Name | `dc1` (The consul default) | | `DisableHostNodeId` | Disable Node Id creation (uses random)| `false` | | `EncryptGossip` | Whether or not gossip is encrypted | `true` | @@ -62,9 +64,27 @@ Alternatively, a YAML file that specifies the values for the parameters can be p ```bash $ helm install --name my-release -f values.yaml stable/consul ``` +> **Tip**: `ConsulConfig` is impossible to set using --set as it's not possible to set list of hashes with it at the moment, use a YAML file instead. > **Tip**: You can use the default [values.yaml](values.yaml) +## Further consul configuration + +To support passing in more detailed/complex configuration options using `secret`s or `configMap`s. As an example, here is what a `values.yaml` could look like: +```yaml +ConsulConfig: + - type: configMap + name: consul-defaults + - type: secret + name: consul-secrets +``` + +> These are both mounted as files in the consul pods, including the secrets. When they are changed, the cluster may need to be restarted. + +> **Important**: Kubernetes does not allow the volumes to be changed for a StatefulSet. If a new item needs to be added to this list, the StatefulSet needs to be deleted and re-created. The contents of each item can change and will be respected when the containers would read configuration (reload/restart). + +This would require the `consul-defaults` `configMap` and `consul-secrets` `secret` in the same `namespace`. There is no difference from the consul perspective, one could use only `secret`s, or only `configMap`s, or neither. They can each contain multiple consul configuration files (every `JSON` file contained in them will be interpreted as one). The order in which the configuration will be loaded is the same order as they are specified in the `ConsulConfig` setting (later overrides earlier). In case they contain multiple files, the order between those files is decided by consul (as per the [--config-dir](https://www.consul.io/docs/agent/options.html#_config_dir) argument in consul agent), but the order in `ConsulConfig` is still respected. The configuration generated by helm (this chart) is loaded last, and therefore overrides the configuration set here. + ## Cleanup orphaned Persistent Volumes Deleting a StateFul will not delete associated Persistent Volumes. @@ -75,6 +95,10 @@ Do the following after deleting the chart release to clean up orphaned Persisten $ kubectl delete pvc -l component=${RELEASE-NAME}-consul ``` +## Pitfalls + +* When ACLs are enabled and `acl_default_policy` is set to `deny`, it is necessary to set the `acl_token` to a token that can perform at least the `consul members`, otherwise the kubernetes liveness probe will keep failing and the containers will be killed every 5 minutes. + ## Testing Helm tests are included and they confirm the first three cluster members have quorum. diff --git a/stable/consul/templates/consul-statefulset.yaml b/stable/consul/templates/consul-statefulset.yaml index 83e9b9df2a..b5d43a5e99 100644 --- a/stable/consul/templates/consul-statefulset.yaml +++ b/stable/consul/templates/consul-statefulset.yaml @@ -107,6 +107,11 @@ spec: - name: gossip-key mountPath: /etc/consul/secrets readOnly: true + {{ range .Values.ConsulConfig }} + - name: userconfig-{{ .name }} + readOnly: true + mountPath: /etc/consul/userconfig/{{ .name }} + {{ end }} livenessProbe: exec: command: @@ -144,6 +149,9 @@ spec: done exec /bin/consul agent \ + {{- range .Values.ConsulConfig }} + -config-dir /etc/consul/userconfig/{{ .name }} \ + {{- end}} {{- if .Values.uiService.enabled }} -ui \ {{- end }} @@ -173,6 +181,15 @@ spec: - name: gossip-key secret: secretName: {{ template "consul.fullname" . }}-gossip-key + {{ range .Values.ConsulConfig }} + - name: userconfig-{{ .name }} + {{ .type }}: + {{- if (eq .type "configMap") }} + name: {{ .name }} + {{- else if (eq .type "secret") }} + secretName: {{ .name }} + {{- end}} + {{ end }} volumeClaimTemplates: - metadata: name: datadir diff --git a/stable/consul/values.yaml b/stable/consul/values.yaml index 7ebb3b596e..2ea77418ba 100644 --- a/stable/consul/values.yaml +++ b/stable/consul/values.yaml @@ -93,6 +93,12 @@ uiService: enabled: true type: "NodePort" +ConsulConfig: [] +# - type: secret +# name: consul-defaults +# - type: configMap +# name: consul-defaults + ## Create an Ingress for the Web UI uiIngress: enabled: false