diff --git a/incubator/kafka/Chart.yaml b/incubator/kafka/Chart.yaml index d391253177..399902a0dd 100755 --- a/incubator/kafka/Chart.yaml +++ b/incubator/kafka/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Apache Kafka is publish-subscribe messaging rethought as a distributed commit log. name: kafka -version: 0.1.2 +version: 0.1.3 keywords: - kafka - zookeeper @@ -14,4 +14,6 @@ sources: maintainers: - name: Faraaz Khan email: faraaz@rationalizeit.us + - name: Marc Villacorta + email: marc.villacorta@gmail.com icon: https://kafka.apache.org/images/logo.png diff --git a/incubator/kafka/README.md b/incubator/kafka/README.md index f078ecf340..977a091c34 100644 --- a/incubator/kafka/README.md +++ b/incubator/kafka/README.md @@ -31,7 +31,7 @@ This chart will do the following: ### Installing the Chart -To install the chart with the release name `my-release` in the default +To install the chart with the release name `my-kafka` in the default namespace: ``` @@ -53,9 +53,8 @@ following configurable parameters: | Parameter | Description | Default | | ----------------------- | ---------------------------------- | ---------------------------------------------------------- | -| `Name` | Kafka master name | `kf` | | `Image` | Kafka Container image name | `solsson/kafka` | -| `ImageTag` | Kafka Container image tag | `0.10.0.1` | +| `ImageTag` | Kafka Container image tag | `0.11.0.0` | | `ImagePullPolicy` | Kafka Container pull policy | `Always` | | `Replicas` | Kafka Brokers | `3` | | `Component` | Kafka k8s selector key | `kafka` | @@ -65,14 +64,13 @@ following configurable parameters: | `MaxMemory` | Kafka container memory limit | `1024Mi` | | `DataDirectory` | Kafka data directory | `/opt/kafka/data` | | `Storage` | Kafka Persistent volume size | `1Gi` | -| `zookeeper.Name` | Name of the Zookeeper Service | `zk` | Specify parameters using `--set key=value[,key=value]` argument to `helm install` Alternatively a YAML file that specifies the values for the parameters can be provided like this: ```bash -$ helm install --name my-release -f values.yaml incubator/kafka +$ helm install --name my-kafka -f values.yaml incubator/kafka ``` ### Connecting to Kafka @@ -88,7 +86,7 @@ metadata: spec: containers: - name: kafka - image: solsson/kafka:0.10.0.1 + image: solsson/kafka:0.11.0.0 command: - sh - -c @@ -98,14 +96,13 @@ spec: Once you have the testclient pod above running, you can list all kafka topics with: -` kubectl exec testclient -- ./bin/kafka-topics.sh --zookeeper -my-release-zk:2181 --list` +` kubectl -n kafka exec -ti testclient -- ./bin/kafka-topics.sh --zookeeper +my-release-zookeeper:2181 --list` Where `my-release` is the name of your helm release. ## Known Limitations -* Namespace creation is not automated * Topic creation is not automated * Only supports storage options that have backends for persistent volume claims (tested mostly on AWS) * Kafka cluster is not accessible via an external endpoint diff --git a/incubator/kafka/notes.txt b/incubator/kafka/notes.txt index 9ced65323f..c91e7466c4 100644 --- a/incubator/kafka/notes.txt +++ b/incubator/kafka/notes.txt @@ -8,7 +8,7 @@ You can connect to Kafka by running a simple pod in the K8s cluster like this wi spec: containers: - name: kafka - image: solsson/kafka:0.10.0.1 + image: solsson/kafka:0.11.0.0 command: - sh - -c @@ -17,20 +17,20 @@ You can connect to Kafka by running a simple pod in the K8s cluster like this wi Once you have the testclient pod above running, you can list all kafka topics with: - kubectl -n kafka exec testclient -- ./bin/kafka-topics.sh --zookeeper my-release-zk:2181 --list + kubectl -n kafka exec testclient -- ./bin/kafka-topics.sh --zookeeper -zookeeper:2181 --list To create a new topic: - kubectl -n kafka exec testclient -- ./bin/kafka-topics.sh --zookeeper my-release-zookeeper-zk:2181 --topic test1 --create --partitions 1 --replication-factor 1 + kubectl -n kafka exec testclient -- ./bin/kafka-topics.sh --zookeeper -zookeeper:2181 --topic test1 --create --partitions 1 --replication-factor 1 To listen for messages on a topic: - kubectl -n kafka exec -ti testclient -- ./bin/kafka-console-consumer.sh --zookeeper my-release-zookeeper-zk:2181 --topic test1 --from-beginning + kubectl -n kafka exec -ti testclient -- ./bin/kafka-console-consumer.sh --bootstrap-server -kafka:9092 --topic test1 --from-beginning To stop the listener session above press: Ctrl+C To start an interactive message producer session: - kubectl -n kafka exec -ti testclient -- ./bin/kafka-console-producer.sh --broker-list my-release-broker-kf.kafka.svc.cluster.local:9092 --topic test1 + kubectl -n kafka exec -ti testclient -- ./bin/kafka-console-producer.sh --broker-list -kafka-headless:9092 --topic test1 To create a message in the above session, simply type the message and press "enter" To end the producer session try: Ctrl+C - Where my-release is the name of your helm release. + Where is the name of your helm release. diff --git a/incubator/kafka/requirements.lock b/incubator/kafka/requirements.lock index a9e31180d5..59b4ec43db 100644 --- a/incubator/kafka/requirements.lock +++ b/incubator/kafka/requirements.lock @@ -1,10 +1,6 @@ dependencies: -- condition: "" - enabled: false - import-values: null - name: zookeeper +- name: zookeeper repository: https://kubernetes-charts-incubator.storage.googleapis.com/ - tags: null - version: 0.2.2 -digest: sha256:7242e077d16e80c449f973c2820629836e06f5be042e2c1fad713c0623af14ef -generated: 2017-05-02T12:18:02.471103601+02:00 + version: 0.3.1 +digest: sha256:a2724fe59862e671ee0b59667422168bf37224d63a3f8fc4340c4fd63d75b646 +generated: 2017-07-29T17:52:16.380268781+02:00 diff --git a/incubator/kafka/requirements.yaml b/incubator/kafka/requirements.yaml index 718467bfc2..3dba0085e3 100644 --- a/incubator/kafka/requirements.yaml +++ b/incubator/kafka/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: zookeeper - version: 0.2.2 + version: 0.3.1 repository: https://kubernetes-charts-incubator.storage.googleapis.com/ diff --git a/incubator/kafka/templates/_helpers.tpl b/incubator/kafka/templates/_helpers.tpl new file mode 100644 index 0000000000..f0d83d2edb --- /dev/null +++ b/incubator/kafka/templates/_helpers.tpl @@ -0,0 +1,16 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/incubator/kafka/templates/kafka-ss.yaml b/incubator/kafka/templates/kafka-ss.yaml deleted file mode 100644 index 98b909d90b..0000000000 --- a/incubator/kafka/templates/kafka-ss.yaml +++ /dev/null @@ -1,128 +0,0 @@ -# headless service for petset DNS -apiVersion: v1 -kind: Service -metadata: - name: "{{ printf "%s-zookeeper-%s" .Release.Name .Values.zookeeper.Name | trunc 63 }}" -spec: - ports: - - port: 2181 - name: client - selector: - component: "{{ .Release.Name }}" - -# A headless service to create DNS records ---- -apiVersion: v1 -kind: Service -metadata: - name: "{{ printf "%s-broker-%s" .Release.Name .Values.Name | trunc 63 }}" - labels: - heritage: {{ .Release.Service | quote }} - release: {{ .Release.Name | quote }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - component: "{{ .Release.Name }}-{{ .Values.Component }}" - annotations: - "helm.sh/created": {{ .Release.Time.Seconds | quote }} - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" -spec: - ports: - - port: 9092 - clusterIP: None - selector: - component: "{{ .Release.Name }}-{{ .Values.Component }}" - app: kafka ---- -apiVersion: v1 -kind: Service -metadata: - name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 63 }}" - labels: - heritage: {{ .Release.Service | quote }} - release: {{ .Release.Name | quote }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - component: "{{ .Release.Name }}-{{ .Values.Component }}" - annotations: - "helm.sh/created": {{ .Release.Time.Seconds | quote }} -spec: - ports: - - port: 9092 - selector: - component: "{{ .Release.Name }}-{{ .Values.Component }}" - app: kafka ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 63 }}" - labels: - app: kafka - heritage: {{ .Release.Service | quote }} - release: {{ .Release.Name | quote }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - component: "{{ .Release.Name }}-{{ .Values.Component }}" - annotations: - "helm.sh/created": {{ .Release.Time.Seconds | quote }} -spec: - serviceName: "{{ printf "%s-broker-%s" .Release.Name .Values.Name | trunc 63 }}" - replicas: {{ default 3 .Values.Replicas }} - template: - metadata: - labels: - app: kafka - component: "{{ .Release.Name }}-{{ .Values.Component }}" - annotations: - pod.alpha.kubernetes.io/initialized: "true" - pod.alpha.kubernetes.io/init-containers: '[ - ]' - spec: - containers: - - name: "{{ printf "%s-broker-%s" .Release.Name .Values.Name | trunc 63 }}" - image: "{{ .Values.Image }}:{{ .Values.ImageTag }}" - imagePullPolicy: "{{ .Values.ImagePullPolicy }}" - livenessProbe: - exec: - command: - - bin/kafka-topics.sh - - --zookeeper - - "{{ printf "%s-zookeeper-%s" .Release.Name .Values.zookeeper.Name | trunc 63 }}:2181" - - --list - initialDelaySeconds: 30 - timeoutSeconds: 5 - name: liveness - readinessProbe: - exec: - command: - - bin/kafka-topics.sh - - --zookeeper - - "{{ printf "%s-zookeeper-%s" .Release.Name .Values.zookeeper.Name | trunc 63 }}:2181" - - --list - initialDelaySeconds: 30 - timeoutSeconds: 5 - name: readiness - ports: - - containerPort: 9092 - name: kafka - resources: - limits: - cpu: "{{ .Values.MaxCpu }}" - memory: "{{ .Values.MaxMemory }}" - requests: - cpu: "{{ .Values.Cpu }}" - memory: "{{ .Values.Memory }}" - command: - - sh - - -c - - "./bin/kafka-server-start.sh config/server.properties --override zookeeper.connect={{ printf "%s-zookeeper-%s" .Release.Name .Values.zookeeper.Name | trunc 63 }}:2181/ --override log.dirs={{ printf "%s/logs" .Values.DataDirectory | trunc 63 }} --override broker.id=${HOSTNAME##*-}" - volumeMounts: - - name: datadir - mountPath: "{{ .Values.DataDirectory }}" - volumeClaimTemplates: - - metadata: - name: datadir - annotations: - volume.alpha.kubernetes.io/storage-class: anything - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: {{ .Values.Storage }} diff --git a/incubator/kafka/templates/service-brokers.yaml b/incubator/kafka/templates/service-brokers.yaml new file mode 100644 index 0000000000..90a8ee82b2 --- /dev/null +++ b/incubator/kafka/templates/service-brokers.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "fullname" . }} + labels: + app: {{ include "name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +spec: + ports: + - name: broker + port: 9092 + selector: + app: {{ include "name" . | quote }} + release: {{ .Release.Name | quote }} diff --git a/incubator/kafka/templates/service-headless.yaml b/incubator/kafka/templates/service-headless.yaml new file mode 100644 index 0000000000..a25df9c416 --- /dev/null +++ b/incubator/kafka/templates/service-headless.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "fullname" . }}-headless + labels: + app: {{ include "name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + ports: + - name: broker + port: 9092 + clusterIP: None + selector: + app: {{ include "name" . | quote }} + release: {{ .Release.Name | quote }} diff --git a/incubator/kafka/templates/statefulset.yaml b/incubator/kafka/templates/statefulset.yaml new file mode 100644 index 0000000000..a2a91fc3af --- /dev/null +++ b/incubator/kafka/templates/statefulset.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: {{ template "fullname" . }} + labels: + app: {{ include "name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +spec: + serviceName: {{ template "fullname" . }}-headless + replicas: {{ default 3 .Values.Replicas }} + template: + metadata: + labels: + app: {{ include "name" . | quote }} + release: {{ .Release.Name | quote }} + spec: + containers: + - name: {{ template "name" . }}-broker + image: "{{ .Values.Image }}:{{ .Values.ImageTag }}" + imagePullPolicy: "{{ .Values.ImagePullPolicy }}" + livenessProbe: + exec: + command: + - bin/kafka-topics.sh + - --zookeeper + - "{{ template "name" . }}-zookeeper:2181" + - --list + initialDelaySeconds: 30 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - bin/kafka-topics.sh + - --zookeeper + - "{{ template "name" . }}-zookeeper:2181" + - --list + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 9092 + name: kafka + resources: + limits: + cpu: "{{ .Values.MaxCpu }}" + memory: "{{ .Values.MaxMemory }}" + requests: + cpu: "{{ .Values.Cpu }}" + memory: "{{ .Values.Memory }}" + command: + - sh + - -c + - "./bin/kafka-server-start.sh config/server.properties --override zookeeper.connect={{ template "name" . }}-zookeeper:2181/ --override broker.id=${HOSTNAME##*-}" + volumeMounts: + - name: datadir + mountPath: "{{ .Values.DataDirectory }}" + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.Storage }} diff --git a/incubator/kafka/values.yaml b/incubator/kafka/values.yaml index 4d6f628fcc..f7d226125b 100644 --- a/incubator/kafka/values.yaml +++ b/incubator/kafka/values.yaml @@ -1,16 +1,38 @@ -Name: "kf" +#------------------------------------------------------------------------------ +# Kafka: +#------------------------------------------------------------------------------ + Replicas: 3 Image: "solsson/kafka" -ImageTag: "0.10.0.1" -ImagePullPolicy: "Always" +ImageTag: "0.11.0.0" +ImagePullPolicy: "IfNotPresent" Cpu: "100m" Memory: "512Mi" MaxCpu: "200m" MaxMemory: "1024Mi" Storage: "1Gi" DataDirectory: "/opt/kafka/data" -Component: "kafka" + +#------------------------------------------------------------------------------ +# Zookeeper: +#------------------------------------------------------------------------------ zookeeper: - Name: "zk" - Component: "zk" + Servers: 3 + Resources: {} + Heap: "1G" + Storage: "2Gi" + ServerPort: 2888 + LeaderElectionPort: 3888 + ClientPort: 2181 + ImagePullPolicy: "IfNotPresent" + TickTimeMs: 2000 + InitTicks: 10 + SyncTicks: 5 + ClientCnxns: 60 + SnapRetain: 3 + PurgeHours: 1 + ProbeInitialDelaySeconds : 15 + ProbeTimeoutSeconds : 5 + AntiAffinity : "hard" + LogLevel : "INFO"