From 3aaf81b5584cdbf6bedfd09fc3a6abd41a39548a Mon Sep 17 00:00:00 2001 From: Levi Thornton Date: Wed, 28 Feb 2018 13:03:47 -0800 Subject: [PATCH] [incubator/kafka] Makes persistence fully configurable (#3721) * Makes kafka persistentVolume configurable * Downgrades version to 0.3.0 * Cleans up * Fixes indentation * Adds missing spaces for helm lint * Goes back to volumeClaimTemplates to support scaling * Adds missing whitespace eater * Follows best practices for defining storageClassName --- incubator/kafka/Chart.yaml | 2 +- incubator/kafka/README.md | 46 ++++++++++++---------- incubator/kafka/templates/configmap.yaml | 2 +- incubator/kafka/templates/statefulset.yaml | 19 +++++++-- incubator/kafka/values.yaml | 36 ++++++++++++----- 5 files changed, 68 insertions(+), 37 deletions(-) diff --git a/incubator/kafka/Chart.yaml b/incubator/kafka/Chart.yaml index 69a06d2dd6..502814c6b1 100755 --- a/incubator/kafka/Chart.yaml +++ b/incubator/kafka/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 description: Apache Kafka is publish-subscribe messaging rethought as a distributed commit log. name: kafka -version: 0.2.15 +version: 0.3.0 keywords: - kafka - zookeeper diff --git a/incubator/kafka/README.md b/incubator/kafka/README.md index 567c84b6fb..fe1c7e5c42 100644 --- a/incubator/kafka/README.md +++ b/incubator/kafka/README.md @@ -51,27 +51,29 @@ This chart includes a ZooKeeper chart as a dependency to the Kafka cluster in its `requirement.yaml` by default. The chart can be customized using the following configurable parameters: -| Parameter | Description | Default | -| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------- | -| `image` | Kafka Container image name | `confluentinc/cp-kafka` | -| `imageTag` | Kafka Container image tag | `4.0.0` | -| `imagePullPolicy` | Kafka Container pull policy | `IfNotPresent` | -| `replicas` | Kafka Brokers | `3` | -| `component` | Kafka k8s selector key | `kafka` | -| `resources` | Kafka resource requests and limits | `{}` | -| `dataDirectory` | Kafka data directory | `/opt/kafka/data` | -| `logSubPath` | Subpath under `dataDirectory` where kafka logs will be placed. `logs/` | `logs` | -| `affinity` | Pod scheduling preferences | `{}` | -| `storage` | Kafka Persistent volume size | `1Gi` | -| `configurationOverrides` | `Kafka ` [configuration setting](https://kafka.apache.org/documentation/#brokerconfigs) overrides in the dictionary format `setting.name: value` | `{ offsets.topic.replication.factor: 3 }` | -| `updateStrategy` | StatefulSet update strategy to use. | `{ type: "OnDelete" }` | -| `zookeeper.enabled` | If True, installs Zookeeper Chart | `true` | -| `zookeeper.resources` | Zookeeper resource requests and limits | `{}` | -| `zookeeper.heap` | JVM heap size to allocate to Zookeeper | `1G` | -| `zookeeper.storage` | Zookeeper Persistent volume size | `2Gi` | -| `zookeeper.imagePullPolicy` | Zookeeper Container pull policy | `IfNotPresent` | -| `zookeeper.url` | URL of Zookeeper Cluster (unneeded if installing Zookeeper Chart) | `""` | -| `zookeeper.port` | Port of Zookeeper Cluster | `2181` | +| Parameter | Description | Default | +| ------------------------------ | --------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- | +| `image` | Kafka Container image name | `confluentinc/cp-kafka` | +| `imageTag` | Kafka Container image tag | `4.0.0` | +| `imagePullPolicy` | Kafka Container pull policy | `IfNotPresent` | +| `replicas` | Kafka Brokers | `3` | +| `component` | Kafka k8s selector key | `kafka` | +| `resources` | Kafka resource requests and limits | `{}` | +| `logSubPath` | Subpath under `persistence.mountPath` where kafka logs will be placed. | `logs` | +| `affinity` | Pod scheduling preferences | `{}` | +| `configurationOverrides` | `Kafka ` [configuration setting][brokerconfigs] overrides in the dictionary format | `{ offsets.topic.replication.factor: 3 }` | +| `updateStrategy` | StatefulSet update strategy to use. | `{ type: "OnDelete" }` | +| `persistence.enabled` | Use a PVC to persist data | `true` | +| `persistence.size` | Size of data volume | `1Gi` | +| `persistence.mountPath` | Mount path of data volume | `/opt/kafka/data` | +| `persistence.storageClass` | Storage class of backing PVC | `nil` | +| `zookeeper.enabled` | If True, installs Zookeeper Chart | `true` | +| `zookeeper.resources` | Zookeeper resource requests and limits | `{}` | +| `zookeeper.heap` | JVM heap size to allocate to Zookeeper | `1G` | +| `zookeeper.storage` | Zookeeper Persistent volume size | `2Gi` | +| `zookeeper.imagePullPolicy` | Zookeeper Container pull policy | `IfNotPresent` | +| `zookeeper.url` | URL of Zookeeper Cluster (unneeded if installing Zookeeper Chart) | `""` | +| `zookeeper.port` | Port of Zookeeper Cluster | `2181` | Specify parameters using `--set key=value[,key=value]` argument to `helm install` @@ -122,3 +124,5 @@ Kafka has a rich ecosystem, with lots of tools. This sections is intended to com * Topic creation is not automated * Only supports storage options that have backends for persistent volume claims (tested mostly on AWS) * Kafka cluster is not accessible via an external endpoint + +[brokerconfigs]: https://kafka.apache.org/documentation/#brokerconfigs diff --git a/incubator/kafka/templates/configmap.yaml b/incubator/kafka/templates/configmap.yaml index 1ccae1c1d5..e739056687 100644 --- a/incubator/kafka/templates/configmap.yaml +++ b/incubator/kafka/templates/configmap.yaml @@ -9,7 +9,7 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" data: zookeeper.connect: "{{ template "zookeeper.url" . }}" - log.dirs: "{{ printf "%s/%s" .Values.dataDirectory .Values.logSubPath }}" + log.dirs: "{{ printf "%s/%s" .Values.persistence.mountPath .Values.logSubPath }}" {{- range $configName, $configValue := .Values.configurationOverrides }} {{ $configName }}: "{{ $configValue -}}" {{- end -}} diff --git a/incubator/kafka/templates/statefulset.yaml b/incubator/kafka/templates/statefulset.yaml index 1f31d076f6..b4f9b09c0f 100644 --- a/incubator/kafka/templates/statefulset.yaml +++ b/incubator/kafka/templates/statefulset.yaml @@ -86,7 +86,13 @@ spec: /etc/confluent/docker/run volumeMounts: - name: datadir - mountPath: "{{ .Values.dataDirectory }}" + mountPath: {{ .Values.persistence.mountPath | quote }} + {{- if not .Values.persistence.enabled }} + volumes: + - name: datadir + emptyDir: {} + {{- end }} + {{- if .Values.persistence.enabled }} volumeClaimTemplates: - metadata: name: datadir @@ -94,7 +100,12 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: {{ .Values.storage }} - {{- if .Values.storageClass }} - storageClassName: {{ .Values.storageClass | quote }} + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + {{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" {{- end }} + {{- end }} + {{- end }} diff --git a/incubator/kafka/values.yaml b/incubator/kafka/values.yaml index 19f2b9579f..e6052b4623 100644 --- a/incubator/kafka/values.yaml +++ b/incubator/kafka/values.yaml @@ -24,10 +24,6 @@ resources: {} # requests: # cpu: 100m # memory: 1024Mi - # -## The size of the persistentVolume to allocate to each Kafka Pod in the StatefulSet. For -## production servers this number should likely be much larger. -storage: "1Gi" ## The StatefulSet Update Strategy which Kafka will use when changes are applied. ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies @@ -37,12 +33,8 @@ updateStrategy: ## The name of the storage class which the cluster should use. # storageClass: default -## The location within the Kafka container where the PV will mount its storage and Kafka will store -## its logs -dataDirectory: "/opt/kafka/data" - -## The subpath within the Kafka container's PV where logs will be stored -## This is combined with `dataDirectory` above, to create, by default: /opt/kafka/data/logs +## The subpath within the Kafka container's PV where logs will be stored. +## This is combined with `persistence.mountPath`, to create, by default: /opt/kafka/data/logs logSubPath: "logs" ## Pod scheduling preferences. @@ -61,6 +53,30 @@ nodeSelector: {} configurationOverrides: "offsets.topic.replication.factor": 3 +## Persistence configuration. Specify if and how to persist data to a persistent volume. +## +persistence: + enabled: true + + ## The size of the PersistentVolume to allocate to each Kafka Pod in the StatefulSet. For + ## production servers this number should likely be much larger. + ## + size: "1Gi" + + ## The location within the Kafka container where the PV will mount its storage and Kafka will + ## store its logs. + ## + mountPath: "/opt/kafka/data" + + ## Kafka data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: + # ------------------------------------------------------------------------------ # Zookeeper: # ------------------------------------------------------------------------------