mirror of
https://github.com/helm/charts.git
synced 2026-08-23 22:37:45 +00:00
Kafka chart use confluent kafka (#2988)
* Use Confluent OSS Kafka Image, Allow user configuration overrides, use a true liveness check * Add Test for Kafka Chart that creates a topic, produces and consumes todays date * Remove superfluous producer line from test
This commit is contained in:
Benjamin Goldberg
committed by
Vic Iglesias
Vic Iglesias
parent
202a7ad8e8
commit
b4ab6998e3
@@ -2,7 +2,7 @@ apiVersion: v1
|
||||
description: Apache Kafka is publish-subscribe messaging rethought as a distributed
|
||||
commit log.
|
||||
name: kafka
|
||||
version: 0.2.5
|
||||
version: 0.2.6
|
||||
keywords:
|
||||
- kafka
|
||||
- zookeeper
|
||||
@@ -11,10 +11,13 @@ home: https://kafka.apache.org/
|
||||
sources:
|
||||
- https://github.com/kubernetes/charts/tree/master/incubator/zookeeper
|
||||
- https://github.com/Yolean/kubernetes-kafka
|
||||
- https://github.com/solsson/dockerfiles/tree/master/kafka
|
||||
- https://github.com/confluentinc/cp-docker-images
|
||||
- https://github.com/apache/kafka
|
||||
maintainers:
|
||||
- name: Faraaz Khan
|
||||
email: faraaz@rationalizeit.us
|
||||
- name: Marc Villacorta
|
||||
email: marc.villacorta@gmail.com
|
||||
- name: Ben Goldberg
|
||||
email: ben@spothero.com
|
||||
icon: https://kafka.apache.org/images/logo.png
|
||||
|
||||
+21
-19
@@ -51,25 +51,27 @@ This chart includes a ZooKeeper chart as a dependency to the Kafka
|
||||
cluster in its `requirement.yaml` by default. The chart can be customized using the
|
||||
following configurable parameters:
|
||||
|
||||
| Parameter | Description | Default |
|
||||
| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- |
|
||||
| `image` | Kafka Container image name | `solsson/kafka` |
|
||||
| `imageTag` | Kafka Container image tag | `1.0.0` |
|
||||
| `imagePullPolicy` | Kafka Container pull policy | `Always` |
|
||||
| `kafkaAntiAffinityEnabled` | If `true`, apply anti-affinity rules between kafka pods. | `true` |
|
||||
| `kafkaAntiAffinity` | If `hard` disallow colocation of Kafka pods, if `soft`, make a best effort-attempt to prevent colocation. | `soft` |
|
||||
| `replicas` | Kafka Brokers | `3` |
|
||||
| `component` | Kafka k8s selector key | `kafka` |
|
||||
| `resources` | Kafka resource requests and limits | `{}` |
|
||||
| `dataDirectory` | Kafka data directory | `/opt/kafka/data` |
|
||||
| `storage` | Kafka Persistent volume size | `1Gi` |
|
||||
| `schema-registry.enabled` | If True, installs Schema Registry Chart | `false` |
|
||||
| `zookeeperAntiAffinityEnabled` | If `true`, apply anti-affinity rules between kafka and zookeeper pods. | `true` |
|
||||
| `zookeeperAntiAffinity` | If `hard` disallow colocation of Kafka and Zookeeper pods, if `soft`, make a best effort-attempt to prevent colocation | `soft` |
|
||||
| `zookeeperAntiAffinityPodName` | Pod Metadata app label of zookeeper pods for anti-affinity rules for use with `In` prefix of affinity rules. | `zookeeper` |
|
||||
| `zookeeper.enabled` | If True, installs Zookeeper Chart | `true` |
|
||||
| `zookeeper.url` | URL of Zookeeper Cluster (unneeded if installing Zookeeper Chart) | `""` |
|
||||
| `zookeeper.port` | Port of Zookeeper Cluster | `2181` |
|
||||
| Parameter | Description | Default |
|
||||
| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------- |
|
||||
| `image` | Kafka Container image name | `solsson/kafka` |
|
||||
| `imageTag` | Kafka Container image tag | `1.0.0` |
|
||||
| `imagePullPolicy` | Kafka Container pull policy | `Always` |
|
||||
| `kafkaAntiAffinityEnabled` | If `true`, apply anti-affinity rules between kafka pods. | `true` |
|
||||
| `kafkaAntiAffinity` | If `hard` disallow colocation of Kafka pods, if `soft`, make a best effort-attempt to prevent colocation. | `soft` |
|
||||
| `replicas` | Kafka Brokers | `3` |
|
||||
| `component` | Kafka k8s selector key | `kafka` |
|
||||
| `resources` | Kafka resource requests and limits | `{}` |
|
||||
| `dataDirectory` | Kafka data directory | `/opt/kafka/data` |
|
||||
| `logSubPath` | Subpath under `dataDirectory` where kafka logs will be placed. `logs/` | `logs` |
|
||||
| `storage` | Kafka Persistent volume size | `1Gi` |
|
||||
| `configurationOverrides` | `Kafka ` [configuration setting](https://kafka.apache.org/documentation/#brokerconfigs) overrides in the dictionary format `setting.name: value` | `{}` |
|
||||
| `schema-registry.enabled` | If True, installs Schema Registry Chart | `false` |
|
||||
| `zookeeperAntiAffinityEnabled` | If `true`, apply anti-affinity rules between kafka and zookeeper pods. | `true` |
|
||||
| `zookeeperAntiAffinity` | If `hard` disallow colocation of Kafka and Zookeeper pods, if `soft`, make a best effort-attempt to prevent colocation | `soft` |
|
||||
| `zookeeperAntiAffinityPodName` | Pod Metadata app label of zookeeper pods for anti-affinity rules for use with `In` prefix of affinity rules. | `zookeeper` |
|
||||
| `zookeeper.enabled` | If True, installs Zookeeper Chart | `true` |
|
||||
| `zookeeper.url` | URL of Zookeeper Cluster (unneeded if installing Zookeeper Chart) | `""` |
|
||||
| `zookeeper.port` | Port of Zookeeper Cluster | `2181` |
|
||||
|
||||
Specify parameters using `--set key=value[,key=value]` argument to `helm install`
|
||||
|
||||
|
||||
@@ -15,6 +15,13 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create the name for our kafka configmap.
|
||||
*/}}
|
||||
{{- define "kafka.configmap" -}}
|
||||
{{- printf "%s-configmap-%d" (include "kafka.fullname" .) .Release.Revision -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Form the Zookeeper URL. If zookeeper is installed as part of this chart, use k8s service discovery,
|
||||
else use user-provided URL
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ template "kafka.configmap" . }}
|
||||
labels:
|
||||
app: "{{ template "kafka.name" . }}"
|
||||
release: {{ .Release.Name | quote }}
|
||||
heritage: {{ .Release.Service | quote }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
|
||||
data:
|
||||
zookeeper.connect: "{{ template "zookeeper.url" . }}"
|
||||
log.dirs: "{{ printf "%s/%s" .Values.dataDirectory .Values.logSubPath }}"
|
||||
{{- range $configName, $configValue := .Values.configurationOverrides }}
|
||||
{{ $configName }}: "{{ $configValue -}}"
|
||||
{{- end -}}
|
||||
@@ -1,3 +1,4 @@
|
||||
{{- $configMapName := include "kafka.configmap" . -}}
|
||||
apiVersion: apps/v1beta1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
@@ -16,29 +17,29 @@ spec:
|
||||
app: {{ include "kafka.name" . | quote }}
|
||||
release: {{ .Release.Name | quote }}
|
||||
spec:
|
||||
{{ if or .Values.kafkaAntiAffinityEnabled .Values.zookeeperAntiAffinityEnabled }}
|
||||
{{- if or .Values.kafkaAntiAffinityEnabled .Values.zookeeperAntiAffinityEnabled }}
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
{{ if or (eq .Values.kafkaAntiAffinity "hard") (eq .Values.zookeeperAntiAffinity "hard") }}
|
||||
{{- if or (eq .Values.kafkaAntiAffinity "hard") (eq .Values.zookeeperAntiAffinity "hard") }}
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
{{ if and .Values.kafkaAntiAffinityEnabled (eq .Values.kafkaAntiAffinity "hard") }}
|
||||
{{- if and .Values.kafkaAntiAffinityEnabled (eq .Values.kafkaAntiAffinity "hard") }}
|
||||
- topologyKey: "kubernetes.io/hostname"
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app: {{ include "kafka.name" . | quote }}
|
||||
release: {{ .Release.Name | quote }}
|
||||
{{ end }}
|
||||
{{ if and .Values.zookeeperAntiAffinityEnabled (eq .Values.zookeeperAntiAffinity "hard") }}
|
||||
{{- end }}
|
||||
{{- if and .Values.zookeeperAntiAffinityEnabled (eq .Values.zookeeperAntiAffinity "hard") }}
|
||||
- topologyKey: "kubernetes.io/hostname"
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app: {{ .Values.zookeeperAntiAffinityPodName }}
|
||||
release: {{ .Release.Name | quote }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
{{ if or (eq .Values.kafkaAntiAffinity "soft") (eq .Values.zookeeperAntiAffinity "soft") }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if or (eq .Values.kafkaAntiAffinity "soft") (eq .Values.zookeeperAntiAffinity "soft") }}
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
{{ if and .Values.kafkaAntiAffinityEnabled (eq .Values.kafkaAntiAffinity "soft") }}
|
||||
{{- if and .Values.kafkaAntiAffinityEnabled (eq .Values.kafkaAntiAffinity "soft") }}
|
||||
- weight: 1
|
||||
podAffinityTerm:
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
@@ -46,34 +47,30 @@ spec:
|
||||
matchLabels:
|
||||
app: {{ include "kafka.name" . | quote }}
|
||||
release: {{ .Release.Name | quote }}
|
||||
{{ end }}
|
||||
{{ if and .Values.zookeeperAntiAffinityEnabled (eq .Values.zookeeperAntiAffinity "soft") }}
|
||||
{{- end }}
|
||||
{{- if and .Values.zookeeperAntiAffinityEnabled (eq .Values.zookeeperAntiAffinity "soft") }}
|
||||
- weight: 1
|
||||
podAffinityTerm:
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app: {{ .Values.zookeeperAntiAffinityPodName }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: {{ template "kafka.name" . }}-broker
|
||||
image: "{{ .Values.image }}:{{ .Values.imageTag }}"
|
||||
imagePullPolicy: "{{ .Values.imagePullPolicy }}"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- bin/kafka-topics.sh
|
||||
- --zookeeper
|
||||
- {{ template "zookeeper.url" . }}
|
||||
- --list
|
||||
tcpSocket:
|
||||
port: 9092
|
||||
initialDelaySeconds: 30
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- bin/kafka-topics.sh
|
||||
- kafka-topics
|
||||
- --zookeeper
|
||||
- {{ template "zookeeper.url" . }}
|
||||
- --list
|
||||
@@ -84,10 +81,39 @@ spec:
|
||||
name: kafka
|
||||
resources:
|
||||
{{ toYaml .Values.resources | indent 10 }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
- name: KAFKA_ZOOKEEPER_CONNECT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: "{{ $configMapName }}"
|
||||
key: zookeeper.connect
|
||||
- name: KAFKA_LOG_DIRS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: "{{ $configMapName }}"
|
||||
key: log.dirs
|
||||
{{- range $configName := (keys .Values.configurationOverrides) }}
|
||||
- name: KAFKA_{{ $configName | replace "." "_" | upper }}
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: "{{ $configMapName }}"
|
||||
key: {{ $configName }}
|
||||
{{- end }}
|
||||
# This is required because the Downward API does not yet support identification of
|
||||
# pod numbering in statefulsets. Thus, we are required to specify a command which
|
||||
# allows us to extract the pod ID for usage as the Kafka Broker ID.
|
||||
# See: https://github.com/kubernetes/kubernetes/issues/31218
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- "./bin/kafka-server-start.sh config/server.properties --override zookeeper.connect={{ template "zookeeper.url" . }}/ --override log.dirs={{ printf "%s/logs" .Values.dataDirectory }} --override broker.id=${HOSTNAME##*-}"
|
||||
- |
|
||||
export KAFKA_BROKER_ID=${HOSTNAME##*-} && \
|
||||
export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://${POD_IP}:9092" && \
|
||||
/etc/confluent/docker/run
|
||||
volumeMounts:
|
||||
- name: datadir
|
||||
mountPath: "{{ .Values.dataDirectory }}"
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: "{{ .Release.Name }}-test-topic-create-consume-produce"
|
||||
annotations:
|
||||
"helm.sh/hook": test-success
|
||||
spec:
|
||||
containers:
|
||||
- name: {{ .Release.Name }}-test-consume
|
||||
image: {{ .Values.image }}:{{ .Values.imageTag }}
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
# Create the topic
|
||||
kafka-topics --zookeeper {{ template "zookeeper.url" . }} --topic helm-test-topic-create-consume-produce --create --partitions 1 --replication-factor 1 --if-not-exists && \
|
||||
# Create a message
|
||||
MESSAGE="`date -u`" && \
|
||||
# Produce a test message to the topic
|
||||
echo $MESSAGE | kafka-console-producer --broker-list {{ template "kafka.fullname" . }}:9092 --topic helm-test-topic-create-consume-produce && \
|
||||
# Consume a test message from the topic
|
||||
kafka-console-consumer --bootstrap-server {{ template "kafka.fullname" . }}-headless:9092 --topic helm-test-topic-create-consume-produce --from-beginning --timeout-ms 2000 | grep "$MESSAGE"
|
||||
restartPolicy: Never
|
||||
@@ -6,10 +6,10 @@
|
||||
replicas: 3
|
||||
|
||||
## The kafka image repository
|
||||
image: "solsson/kafka"
|
||||
image: "confluentinc/cp-kafka"
|
||||
|
||||
## The kafka image tag
|
||||
imageTag: "1.0.0"
|
||||
imageTag: "4.0.0"
|
||||
|
||||
## Specify a imagePullPolicy
|
||||
## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
@@ -36,6 +36,10 @@ storage: "1Gi"
|
||||
## its logs
|
||||
dataDirectory: "/opt/kafka/data"
|
||||
|
||||
## The subpath within the Kafka container's PV where logs will be stored
|
||||
## This is combined with `dataDirectory` above, to create, by default: /opt/kafka/data/logs
|
||||
logSubPath: "logs"
|
||||
|
||||
## Attempt to prevent Kafka pods from being colocated with eachother.
|
||||
kafkaAntiAffinityEnabled: true
|
||||
## Prevent Kafka pods from being colocated on the same node.
|
||||
@@ -59,6 +63,11 @@ zookeeperAntiAffinity: "soft"
|
||||
## indicate which zookeeper pods should be filtered.
|
||||
zookeeperAntiAffinityPodName: "zookeeper"
|
||||
|
||||
## Configuration Overrides. Specify any Kafka settings you would like set on the StatefulSet
|
||||
## here in map format, as defined in the official docs:
|
||||
## ref: https://kafka.apache.org/documentation/#brokerconfigs
|
||||
configurationOverrides: {}
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Zookeeper:
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user