diff --git a/incubator/kafka/.helmignore b/incubator/kafka/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/incubator/kafka/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/incubator/kafka/Chart.yaml b/incubator/kafka/Chart.yaml new file mode 100755 index 0000000000..0c50b75327 --- /dev/null +++ b/incubator/kafka/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +description: A Kafka Helm chart for Kubernetes +name: kafka +version: 0.1.0 +keywords: + - kafka + - zookeeper + - kafka petset +home: https://kafka.apache.org/ +sources: + - https://github.com/kubernetes/charts/tree/master/incubator/zookeeper + - https://github.com/Yolean/kubernetes-kafka + - https://github.com/solsson/dockerfiles/tree/master/kafka +maintainers: + - name: Faraaz Khan + email: faraaz@rationalizeit.us +icon: https://kafka.apache.org/images/logo.png diff --git a/incubator/kafka/README.md b/incubator/kafka/README.md new file mode 100644 index 0000000000..d0b0f6b7a5 --- /dev/null +++ b/incubator/kafka/README.md @@ -0,0 +1,113 @@ +# Apache Kafka Helm Chart + +This is an implementation of Kafka PetSet found here: + + * https://github.com/Yolean/kubernetes-kafka + +## Pre Requisites: + +* Kubernetes 1.3 with alpha APIs enabled and support for storage classes + +* PV support on underlying infrastructure + +* Requires at least `v2.0.0-beta.1` version of helm to support + dependency management with requirements.yaml + +## PetSet Details + +* http://kubernetes.io/docs/user-guide/petset/ + +## PetSet Caveats + +* http://kubernetes.io/docs/user-guide/petset/#alpha-limitations + +## Chart Details + +This chart will do the following: + +* Implement a dynamically scalable kafka cluster using Kubernetes + PetSets + +* Implement a dynamically scalable zookeeper cluster as another Kubernetes PetSet required for the Kafka cluster above + +### Installing the Chart + +To install the chart with the release name `my-release` in the default +namespace: + +``` +helm repo add incubator +http://storage.googleapis.com/kubernetes-charts-incubator +helm install --name my-kafka incubator/kafka +``` + +If using a dedicated namespace(recommended) then make sure the namespace +exists with: + +``` +kubectl create ns kafka +helm install --name my-kafka --set global.namespace=kafka incubator/kafka +``` + +This chart includes a ZooKeeper chart as a dependency to the Kafka +cluster in its `requirement.yaml`. The chart can be customized using the +following configurable parameters: + +| Parameter | Description | Default | +| ----------------------- | ---------------------------------- | ---------------------------------------------------------- | +| `Name` | Kafka master name | `kf` | +| `Image` | Kafka Container image name | `solsson/kafka` | +| `ImageTag` | Kafka Container image tag | `0.10.0.1` | +| `ImagePullPolicy` | Kafka Container pull policy | `Always` | +| `Replicas` | Kafka Brokers | `3` | +| `Component` | Kafka k8s selector key | `kafka` | +| `Cpu` | Kafka container requested cpu | `100m` | +| `Memory` | Kafka container requested memory | `512Mi` | +| `MaxCpu` | Kafka container cpu limit | `200m` | +| `MaxMemory` | Kafka container memory limit | `1024Mi` | +| `DataDirectory` | Kafka data directory | `/opt/kafka/data` | +| `Storage` | Kafka Persistent volume size | `1Gi` | +| `zookeeper.Name` | Name of the Zookeeper Service | `zk` | + +Specify parameters using `--set key=value[,key=value]` argument to `helm install` + +Alternatively a YAML file that specifies the values for the parameters can be provided like this: + +```bash +$ helm install --name my-release -f values.yaml incubator/kafka +``` + +### Connecting to Kafka + +You can connect to Kafka by running a simple pod in the K8s cluster like this with a configuration like this: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: testclient + namespace: kafka +spec: + containers: + - name: kafka + image: solsson/kafka:0.10.0.1 + command: + - sh + - -c + - "exec tail -f /dev/null" +``` + +Once you have the testclient pod above running, you can list all kafka +topics with: + +` kubectl exec testclient -- ./bin/kafka-topics.sh --zookeeper +my-release-zk:2181 --list` + +Where `my-release` is the name of your helm release. + +## Known Limitations + +* Namespace creation is not automated +* Topic creation is not automated +* Only supports storage options that have backends for persistent volume claims (tested mostly on AWS) +* Kafka cluster is not accessible via an external endpoint diff --git a/incubator/kafka/notes.txt b/incubator/kafka/notes.txt new file mode 100644 index 0000000000..9ced65323f --- /dev/null +++ b/incubator/kafka/notes.txt @@ -0,0 +1,36 @@ +You can connect to Kafka by running a simple pod in the K8s cluster like this with a configuration like this: + + apiVersion: v1 + kind: Pod + metadata: + name: testclient + namespace: kafka + spec: + containers: + - name: kafka + image: solsson/kafka:0.10.0.1 + command: + - sh + - -c + - "exec tail -f /dev/null" + + Once you have the testclient pod above running, you can list all kafka + topics with: + + kubectl -n kafka exec testclient -- ./bin/kafka-topics.sh --zookeeper my-release-zk:2181 --list + + To create a new topic: + kubectl -n kafka exec testclient -- ./bin/kafka-topics.sh --zookeeper my-release-zookeeper-zk:2181 --topic test1 --create --partitions 1 --replication-factor 1 + + To listen for messages on a topic: + kubectl -n kafka exec -ti testclient -- ./bin/kafka-console-consumer.sh --zookeeper my-release-zookeeper-zk:2181 --topic test1 --from-beginning + + To stop the listener session above press: Ctrl+C + + To start an interactive message producer session: + kubectl -n kafka exec -ti testclient -- ./bin/kafka-console-producer.sh --broker-list my-release-broker-kf.kafka.svc.cluster.local:9092 --topic test1 + + To create a message in the above session, simply type the message and press "enter" + To end the producer session try: Ctrl+C + + Where my-release is the name of your helm release. diff --git a/incubator/kafka/requirements.lock b/incubator/kafka/requirements.lock new file mode 100644 index 0000000000..03d1567008 --- /dev/null +++ b/incubator/kafka/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: zookeeper + repository: http://storage.googleapis.com/kubernetes-charts-incubator + version: 0.1.1 +digest: sha256:8210ab022b7aaac63d45df4db7deb770bf08c5d7e24c924764648c55be5e0905 +generated: 2016-10-23T22:25:39.161124014-05:00 diff --git a/incubator/kafka/requirements.yaml b/incubator/kafka/requirements.yaml new file mode 100644 index 0000000000..675b2c4002 --- /dev/null +++ b/incubator/kafka/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: +- name: zookeeper + version: 0.1.1 + repository: http://storage.googleapis.com/kubernetes-charts-incubator diff --git a/incubator/kafka/templates/kafka-petset.yaml b/incubator/kafka/templates/kafka-petset.yaml new file mode 100644 index 0000000000..61fd572a7a --- /dev/null +++ b/incubator/kafka/templates/kafka-petset.yaml @@ -0,0 +1,129 @@ +--- +# headless service for petset DNS +apiVersion: v1 +kind: Service +metadata: + name: "{{ printf "%s-zookeeper-%s" .Release.Name .Values.zookeeper.Name | trunc 63 }}" +spec: + ports: + - port: 2181 + name: client + selector: + component: "{{.Release.Name}}-{{.Values.zookeeper.Component}}" + +# A headless service to create DNS records +--- +apiVersion: v1 +kind: Service +metadata: + name: "{{ printf "%s-broker-%s" .Release.Name .Values.Name | trunc 63 }}" + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + component: "{{.Release.Name}}-{{.Values.Component}}" + annotations: + "helm.sh/created": {{.Release.Time.Seconds | quote }} + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + ports: + - port: 9092 + clusterIP: None + selector: + component: "{{.Release.Name}}-{{.Values.Component}}" + app: kafka +--- +apiVersion: v1 +kind: Service +metadata: + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 63 }}" + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + component: "{{.Release.Name}}-{{.Values.Component}}" + annotations: + "helm.sh/created": {{.Release.Time.Seconds | quote }} +spec: + ports: + - port: 9092 + selector: + component: "{{.Release.Name}}-{{.Values.Component}}" + app: kafka +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: "{{ printf "%s-%s" .Release.Name .Values.Name | trunc 63 }}" + labels: + app: kafka + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + component: "{{.Release.Name}}-{{.Values.Component}}" + annotations: + "helm.sh/created": {{.Release.Time.Seconds | quote }} +spec: + serviceName: "{{ printf "%s-broker-%s" .Release.Name .Values.Name | trunc 63 }}" + replicas: {{ default 3 .Values.Replicas }} + template: + metadata: + labels: + app: kafka + component: "{{.Release.Name}}-{{.Values.Component}}" + annotations: + pod.alpha.kubernetes.io/initialized: "true" + pod.alpha.kubernetes.io/init-containers: '[ + ]' + spec: + containers: + - name: "{{ printf "%s-broker-%s" .Release.Name .Values.Name | trunc 63 }}" + image: "{{.Values.Image}}:{{.Values.ImageTag}}" + imagePullPolicy: "{{.Values.ImagePullPolicy}}" + livenessProbe: + exec: + command: + - bin/kafka-topics.sh + - --zookeeper + - "{{ printf "%s-%s" .Release.Name .Values.zookeeper.Name | trunc 63 }}:2181" + - --list + initialDelaySeconds: 30 + timeoutSeconds: 5 + name: liveness + readinessProbe: + exec: + command: + - bin/kafka-topics.sh + - --zookeeper + - "{{ printf "%s-%s" .Release.Name .Values.zookeeper.Name | trunc 63 }}:2181" + - --list + initialDelaySeconds: 30 + timeoutSeconds: 5 + name: readiness + ports: + - containerPort: 9092 + name: kafka + resources: + limits: + cpu: "{{ .Values.MaxCpu }}" + memory: "{{ .Values.MaxMemory }}" + requests: + cpu: "{{ .Values.Cpu }}" + memory: "{{ .Values.Memory }}" + command: + - sh + - -c + - "./bin/kafka-server-start.sh config/server.properties --override zookeeper.connect={{ printf "%s-%s" .Release.Name .Values.zookeeper.Name | trunc 63 }}:2181/ --override broker.id=${HOSTNAME##*-}" + volumeMounts: + - name: datadir + mountPath: "{{ .Values.DataDirectory }}" + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{.Values.Storage}} diff --git a/incubator/kafka/values.yaml b/incubator/kafka/values.yaml new file mode 100644 index 0000000000..4d6f628fcc --- /dev/null +++ b/incubator/kafka/values.yaml @@ -0,0 +1,16 @@ +Name: "kf" +Replicas: 3 +Image: "solsson/kafka" +ImageTag: "0.10.0.1" +ImagePullPolicy: "Always" +Cpu: "100m" +Memory: "512Mi" +MaxCpu: "200m" +MaxMemory: "1024Mi" +Storage: "1Gi" +DataDirectory: "/opt/kafka/data" +Component: "kafka" + +zookeeper: + Name: "zk" + Component: "zk"