From b4ab6998e3fee58733041b95ebbf239ff3f4f70f Mon Sep 17 00:00:00 2001 From: Benjamin Goldberg Date: Mon, 11 Dec 2017 13:46:25 -0600 Subject: [PATCH] Kafka chart use confluent kafka (#2988) * Use Confluent OSS Kafka Image, Allow user configuration overrides, use a true liveness check * Add Test for Kafka Chart that creates a topic, produces and consumes todays date * Remove superfluous producer line from test --- incubator/kafka/Chart.yaml | 7 +- incubator/kafka/README.md | 40 ++++++----- incubator/kafka/templates/_helpers.tpl | 7 ++ incubator/kafka/templates/configmap.yaml | 15 ++++ incubator/kafka/templates/statefulset.yaml | 70 +++++++++++++------ .../test_topic_create_consume_produce.yaml | 23 ++++++ incubator/kafka/values.yaml | 13 +++- 7 files changed, 130 insertions(+), 45 deletions(-) create mode 100644 incubator/kafka/templates/configmap.yaml create mode 100644 incubator/kafka/templates/tests/test_topic_create_consume_produce.yaml diff --git a/incubator/kafka/Chart.yaml b/incubator/kafka/Chart.yaml index 0cd2168ae4..fb626c9f4c 100755 --- a/incubator/kafka/Chart.yaml +++ b/incubator/kafka/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 description: Apache Kafka is publish-subscribe messaging rethought as a distributed commit log. name: kafka -version: 0.2.5 +version: 0.2.6 keywords: - kafka - zookeeper @@ -11,10 +11,13 @@ home: https://kafka.apache.org/ sources: - https://github.com/kubernetes/charts/tree/master/incubator/zookeeper - https://github.com/Yolean/kubernetes-kafka -- https://github.com/solsson/dockerfiles/tree/master/kafka +- https://github.com/confluentinc/cp-docker-images +- https://github.com/apache/kafka maintainers: - name: Faraaz Khan email: faraaz@rationalizeit.us - name: Marc Villacorta email: marc.villacorta@gmail.com +- name: Ben Goldberg + email: ben@spothero.com icon: https://kafka.apache.org/images/logo.png diff --git a/incubator/kafka/README.md b/incubator/kafka/README.md index 1a23385cc1..8d3575455e 100644 --- a/incubator/kafka/README.md +++ b/incubator/kafka/README.md @@ -51,25 +51,27 @@ This chart includes a ZooKeeper chart as a dependency to the Kafka cluster in its `requirement.yaml` by default. The chart can be customized using the following configurable parameters: -| Parameter | Description | Default | -| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- | -| `image` | Kafka Container image name | `solsson/kafka` | -| `imageTag` | Kafka Container image tag | `1.0.0` | -| `imagePullPolicy` | Kafka Container pull policy | `Always` | -| `kafkaAntiAffinityEnabled` | If `true`, apply anti-affinity rules between kafka pods. | `true` | -| `kafkaAntiAffinity` | If `hard` disallow colocation of Kafka pods, if `soft`, make a best effort-attempt to prevent colocation. | `soft` | -| `replicas` | Kafka Brokers | `3` | -| `component` | Kafka k8s selector key | `kafka` | -| `resources` | Kafka resource requests and limits | `{}` | -| `dataDirectory` | Kafka data directory | `/opt/kafka/data` | -| `storage` | Kafka Persistent volume size | `1Gi` | -| `schema-registry.enabled` | If True, installs Schema Registry Chart | `false` | -| `zookeeperAntiAffinityEnabled` | If `true`, apply anti-affinity rules between kafka and zookeeper pods. | `true` | -| `zookeeperAntiAffinity` | If `hard` disallow colocation of Kafka and Zookeeper pods, if `soft`, make a best effort-attempt to prevent colocation | `soft` | -| `zookeeperAntiAffinityPodName` | Pod Metadata app label of zookeeper pods for anti-affinity rules for use with `In` prefix of affinity rules. | `zookeeper` | -| `zookeeper.enabled` | If True, installs Zookeeper Chart | `true` | -| `zookeeper.url` | URL of Zookeeper Cluster (unneeded if installing Zookeeper Chart) | `""` | -| `zookeeper.port` | Port of Zookeeper Cluster | `2181` | +| Parameter | Description | Default | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------- | +| `image` | Kafka Container image name | `solsson/kafka` | +| `imageTag` | Kafka Container image tag | `1.0.0` | +| `imagePullPolicy` | Kafka Container pull policy | `Always` | +| `kafkaAntiAffinityEnabled` | If `true`, apply anti-affinity rules between kafka pods. | `true` | +| `kafkaAntiAffinity` | If `hard` disallow colocation of Kafka pods, if `soft`, make a best effort-attempt to prevent colocation. | `soft` | +| `replicas` | Kafka Brokers | `3` | +| `component` | Kafka k8s selector key | `kafka` | +| `resources` | Kafka resource requests and limits | `{}` | +| `dataDirectory` | Kafka data directory | `/opt/kafka/data` | +| `logSubPath` | Subpath under `dataDirectory` where kafka logs will be placed. `logs/` | `logs` | +| `storage` | Kafka Persistent volume size | `1Gi` | +| `configurationOverrides` | `Kafka ` [configuration setting](https://kafka.apache.org/documentation/#brokerconfigs) overrides in the dictionary format `setting.name: value` | `{}` | +| `schema-registry.enabled` | If True, installs Schema Registry Chart | `false` | +| `zookeeperAntiAffinityEnabled` | If `true`, apply anti-affinity rules between kafka and zookeeper pods. | `true` | +| `zookeeperAntiAffinity` | If `hard` disallow colocation of Kafka and Zookeeper pods, if `soft`, make a best effort-attempt to prevent colocation | `soft` | +| `zookeeperAntiAffinityPodName` | Pod Metadata app label of zookeeper pods for anti-affinity rules for use with `In` prefix of affinity rules. | `zookeeper` | +| `zookeeper.enabled` | If True, installs Zookeeper Chart | `true` | +| `zookeeper.url` | URL of Zookeeper Cluster (unneeded if installing Zookeeper Chart) | `""` | +| `zookeeper.port` | Port of Zookeeper Cluster | `2181` | Specify parameters using `--set key=value[,key=value]` argument to `helm install` diff --git a/incubator/kafka/templates/_helpers.tpl b/incubator/kafka/templates/_helpers.tpl index 8cc8c531d4..94c7674fb5 100644 --- a/incubator/kafka/templates/_helpers.tpl +++ b/incubator/kafka/templates/_helpers.tpl @@ -15,6 +15,13 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create the name for our kafka configmap. +*/}} +{{- define "kafka.configmap" -}} +{{- printf "%s-configmap-%d" (include "kafka.fullname" .) .Release.Revision -}} +{{- end -}} + {{/* Form the Zookeeper URL. If zookeeper is installed as part of this chart, use k8s service discovery, else use user-provided URL diff --git a/incubator/kafka/templates/configmap.yaml b/incubator/kafka/templates/configmap.yaml new file mode 100644 index 0000000000..1ccae1c1d5 --- /dev/null +++ b/incubator/kafka/templates/configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kafka.configmap" . }} + labels: + app: "{{ template "kafka.name" . }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +data: + zookeeper.connect: "{{ template "zookeeper.url" . }}" + log.dirs: "{{ printf "%s/%s" .Values.dataDirectory .Values.logSubPath }}" +{{- range $configName, $configValue := .Values.configurationOverrides }} + {{ $configName }}: "{{ $configValue -}}" +{{- end -}} diff --git a/incubator/kafka/templates/statefulset.yaml b/incubator/kafka/templates/statefulset.yaml index a17c2f2d0e..a9452d8617 100644 --- a/incubator/kafka/templates/statefulset.yaml +++ b/incubator/kafka/templates/statefulset.yaml @@ -1,3 +1,4 @@ +{{- $configMapName := include "kafka.configmap" . -}} apiVersion: apps/v1beta1 kind: StatefulSet metadata: @@ -16,29 +17,29 @@ spec: app: {{ include "kafka.name" . | quote }} release: {{ .Release.Name | quote }} spec: - {{ if or .Values.kafkaAntiAffinityEnabled .Values.zookeeperAntiAffinityEnabled }} + {{- if or .Values.kafkaAntiAffinityEnabled .Values.zookeeperAntiAffinityEnabled }} affinity: podAntiAffinity: - {{ if or (eq .Values.kafkaAntiAffinity "hard") (eq .Values.zookeeperAntiAffinity "hard") }} + {{- if or (eq .Values.kafkaAntiAffinity "hard") (eq .Values.zookeeperAntiAffinity "hard") }} requiredDuringSchedulingIgnoredDuringExecution: - {{ if and .Values.kafkaAntiAffinityEnabled (eq .Values.kafkaAntiAffinity "hard") }} + {{- if and .Values.kafkaAntiAffinityEnabled (eq .Values.kafkaAntiAffinity "hard") }} - topologyKey: "kubernetes.io/hostname" labelSelector: matchLabels: app: {{ include "kafka.name" . | quote }} release: {{ .Release.Name | quote }} - {{ end }} - {{ if and .Values.zookeeperAntiAffinityEnabled (eq .Values.zookeeperAntiAffinity "hard") }} + {{- end }} + {{- if and .Values.zookeeperAntiAffinityEnabled (eq .Values.zookeeperAntiAffinity "hard") }} - topologyKey: "kubernetes.io/hostname" labelSelector: matchLabels: app: {{ .Values.zookeeperAntiAffinityPodName }} release: {{ .Release.Name | quote }} - {{ end }} - {{ end }} - {{ if or (eq .Values.kafkaAntiAffinity "soft") (eq .Values.zookeeperAntiAffinity "soft") }} + {{- end }} + {{- end }} + {{- if or (eq .Values.kafkaAntiAffinity "soft") (eq .Values.zookeeperAntiAffinity "soft") }} preferredDuringSchedulingIgnoredDuringExecution: - {{ if and .Values.kafkaAntiAffinityEnabled (eq .Values.kafkaAntiAffinity "soft") }} + {{- if and .Values.kafkaAntiAffinityEnabled (eq .Values.kafkaAntiAffinity "soft") }} - weight: 1 podAffinityTerm: topologyKey: "kubernetes.io/hostname" @@ -46,34 +47,30 @@ spec: matchLabels: app: {{ include "kafka.name" . | quote }} release: {{ .Release.Name | quote }} - {{ end }} - {{ if and .Values.zookeeperAntiAffinityEnabled (eq .Values.zookeeperAntiAffinity "soft") }} + {{- end }} + {{- if and .Values.zookeeperAntiAffinityEnabled (eq .Values.zookeeperAntiAffinity "soft") }} - weight: 1 podAffinityTerm: topologyKey: "kubernetes.io/hostname" labelSelector: matchLabels: app: {{ .Values.zookeeperAntiAffinityPodName }} - {{ end }} - {{ end }} - {{ end }} + {{- end }} + {{- end }} + {{- end }} containers: - name: {{ template "kafka.name" . }}-broker image: "{{ .Values.image }}:{{ .Values.imageTag }}" imagePullPolicy: "{{ .Values.imagePullPolicy }}" livenessProbe: - exec: - command: - - bin/kafka-topics.sh - - --zookeeper - - {{ template "zookeeper.url" . }} - - --list + tcpSocket: + port: 9092 initialDelaySeconds: 30 timeoutSeconds: 5 readinessProbe: exec: command: - - bin/kafka-topics.sh + - kafka-topics - --zookeeper - {{ template "zookeeper.url" . }} - --list @@ -84,10 +81,39 @@ spec: name: kafka resources: {{ toYaml .Values.resources | indent 10 }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KAFKA_ZOOKEEPER_CONNECT + valueFrom: + configMapKeyRef: + name: "{{ $configMapName }}" + key: zookeeper.connect + - name: KAFKA_LOG_DIRS + valueFrom: + configMapKeyRef: + name: "{{ $configMapName }}" + key: log.dirs + {{- range $configName := (keys .Values.configurationOverrides) }} + - name: KAFKA_{{ $configName | replace "." "_" | upper }} + valueFrom: + configMapKeyRef: + name: "{{ $configMapName }}" + key: {{ $configName }} + {{- end }} + # This is required because the Downward API does not yet support identification of + # pod numbering in statefulsets. Thus, we are required to specify a command which + # allows us to extract the pod ID for usage as the Kafka Broker ID. + # See: https://github.com/kubernetes/kubernetes/issues/31218 command: - sh - -c - - "./bin/kafka-server-start.sh config/server.properties --override zookeeper.connect={{ template "zookeeper.url" . }}/ --override log.dirs={{ printf "%s/logs" .Values.dataDirectory }} --override broker.id=${HOSTNAME##*-}" + - | + export KAFKA_BROKER_ID=${HOSTNAME##*-} && \ + export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://${POD_IP}:9092" && \ + /etc/confluent/docker/run volumeMounts: - name: datadir mountPath: "{{ .Values.dataDirectory }}" diff --git a/incubator/kafka/templates/tests/test_topic_create_consume_produce.yaml b/incubator/kafka/templates/tests/test_topic_create_consume_produce.yaml new file mode 100644 index 0000000000..cff98b1979 --- /dev/null +++ b/incubator/kafka/templates/tests/test_topic_create_consume_produce.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ .Release.Name }}-test-topic-create-consume-produce" + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: {{ .Release.Name }}-test-consume + image: {{ .Values.image }}:{{ .Values.imageTag }} + command: + - sh + - -c + - | + # Create the topic + kafka-topics --zookeeper {{ template "zookeeper.url" . }} --topic helm-test-topic-create-consume-produce --create --partitions 1 --replication-factor 1 --if-not-exists && \ + # Create a message + MESSAGE="`date -u`" && \ + # Produce a test message to the topic + echo $MESSAGE | kafka-console-producer --broker-list {{ template "kafka.fullname" . }}:9092 --topic helm-test-topic-create-consume-produce && \ + # Consume a test message from the topic + kafka-console-consumer --bootstrap-server {{ template "kafka.fullname" . }}-headless:9092 --topic helm-test-topic-create-consume-produce --from-beginning --timeout-ms 2000 | grep "$MESSAGE" + restartPolicy: Never diff --git a/incubator/kafka/values.yaml b/incubator/kafka/values.yaml index 8ae28667da..5b53fe360d 100644 --- a/incubator/kafka/values.yaml +++ b/incubator/kafka/values.yaml @@ -6,10 +6,10 @@ replicas: 3 ## The kafka image repository -image: "solsson/kafka" +image: "confluentinc/cp-kafka" ## The kafka image tag -imageTag: "1.0.0" +imageTag: "4.0.0" ## Specify a imagePullPolicy ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -36,6 +36,10 @@ storage: "1Gi" ## its logs dataDirectory: "/opt/kafka/data" +## The subpath within the Kafka container's PV where logs will be stored +## This is combined with `dataDirectory` above, to create, by default: /opt/kafka/data/logs +logSubPath: "logs" + ## Attempt to prevent Kafka pods from being colocated with eachother. kafkaAntiAffinityEnabled: true ## Prevent Kafka pods from being colocated on the same node. @@ -59,6 +63,11 @@ zookeeperAntiAffinity: "soft" ## indicate which zookeeper pods should be filtered. zookeeperAntiAffinityPodName: "zookeeper" +## Configuration Overrides. Specify any Kafka settings you would like set on the StatefulSet +## here in map format, as defined in the official docs: +## ref: https://kafka.apache.org/documentation/#brokerconfigs +configurationOverrides: {} + # ------------------------------------------------------------------------------ # Zookeeper: # ------------------------------------------------------------------------------