github/deprecated-helm-charts
1
0
Fork 0
mirror of https://github.com/helm/charts.git synced 2026-08-23 22:37:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

[incubator/kafka] Makes affinities fully configurable (#3146)

Browse Source 
* Makes kafka (+ child zookeeper) affinities fully configurable

* Tests kafka consumer via broker svc rather than headless svc

* Consumes single message during kafka consumer test

* Reverts kafka consumer test to use headless svc

* Debugs kafka consumer test via stdout

* Reverts kafka consumer test stdout debugging

* Attempts to fix kafka consumer test
This commit is contained in:
Federico Nusymowicz
2018-01-07 06:47:10 -08:00
committed by k8s-ci-robot
parent a385adf140
commit d035ccb2d7
7 changed files with 15 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
incubator/kafka/Chart.yaml
+1 -1
View File
@@ -2,7 +2,7 @@ apiVersion: v1
description: Apache Kafka is publish-subscribe messaging rethought as a distributed
commit log.
name: kafka
version: 0.2.7
version: 0.2.8
keywords:
- kafka
- zookeeper
incubator/kafka/README.md
+1 -5
View File
@@ -56,19 +56,15 @@ following configurable parameters:
| `image` | Kafka Container image name | `solsson/kafka` |
| `imageTag` | Kafka Container image tag | `1.0.0` |
| `imagePullPolicy` | Kafka Container pull policy | `Always` |
| `kafkaAntiAffinityEnabled` | If `true`, apply anti-affinity rules between kafka pods. | `true` |
| `kafkaAntiAffinity` | If `hard` disallow colocation of Kafka pods, if `soft`, make a best effort-attempt to prevent colocation. | `soft` |
| `replicas` | Kafka Brokers | `3` |
| `component` | Kafka k8s selector key | `kafka` |
| `resources` | Kafka resource requests and limits | `{}` |
| `dataDirectory` | Kafka data directory | `/opt/kafka/data` |
| `logSubPath` | Subpath under `dataDirectory` where kafka logs will be placed. `logs/` | `logs` |
| `affinity` | Pod scheduling preferences | `{}` |
| `storage` | Kafka Persistent volume size | `1Gi` |
| `configurationOverrides` | `Kafka ` [configuration setting](https://kafka.apache.org/documentation/#brokerconfigs) overrides in the dictionary format `setting.name: value` | `{}` |
| `schema-registry.enabled` | If True, installs Schema Registry Chart | `false` |
| `zookeeperAntiAffinityEnabled` | If `true`, apply anti-affinity rules between kafka and zookeeper pods. | `true` |
| `zookeeperAntiAffinity` | If `hard` disallow colocation of Kafka and Zookeeper pods, if `soft`, make a best effort-attempt to prevent colocation | `soft` |
| `zookeeperAntiAffinityPodName` | Pod Metadata app label of zookeeper pods for anti-affinity rules for use with `In` prefix of affinity rules. | `zookeeper` |
| `zookeeper.enabled` | If True, installs Zookeeper Chart | `true` |
| `zookeeper.url` | URL of Zookeeper Cluster (unneeded if installing Zookeeper Chart) | `""` |
| `zookeeper.port` | Port of Zookeeper Cluster | `2181` |
incubator/kafka/requirements.lock
+3 -3
View File
@@ -1,9 +1,9 @@
dependencies:
- name: zookeeper
repository: https://kubernetes-charts-incubator.storage.googleapis.com/
version: 0.4.2
version: 0.4.3
- name: schema-registry
repository: https://kubernetes-charts-incubator.storage.googleapis.com/
version: 0.1.0
digest: sha256:0591abba1e4a7ee22d5eb6b8f573fa40affe62f53c954a6399e32f5609ce968e
generated: 2017-12-04T08:35:47.44854-06:00
digest: sha256:44e2a18583a8ecc8c92ccf042f69f1b656c2e06b01bef6fb03e289b2d511c7fc
generated: 2017-12-23T13:47:40.667268-08:00
incubator/kafka/requirements.yaml
+1 -1
View File
@@ -1,6 +1,6 @@
dependencies:
- name: zookeeper
version: 0.4.2
version: 0.4.3
repository: https://kubernetes-charts-incubator.storage.googleapis.com/
condition: zookeeper.enabled
- name: schema-registry
incubator/kafka/templates/statefulset.yaml
+3 -40
View File
@@ -17,47 +17,10 @@ spec:
app: {{ include "kafka.name" . | quote }}
release: {{ .Release.Name | quote }}
spec:
{{- if or .Values.kafkaAntiAffinityEnabled .Values.zookeeperAntiAffinityEnabled }}
{{- if .Values.affinity }}
affinity:
podAntiAffinity:
{{- if or (eq .Values.kafkaAntiAffinity "hard") (eq .Values.zookeeperAntiAffinity "hard") }}
requiredDuringSchedulingIgnoredDuringExecution:
{{- if and .Values.kafkaAntiAffinityEnabled (eq .Values.kafkaAntiAffinity "hard") }}
- topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ include "kafka.name" . | quote }}
release: {{ .Release.Name | quote }}
{{- end }}
{{- if and .Values.zookeeperAntiAffinityEnabled (eq .Values.zookeeperAntiAffinity "hard") }}
- topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ .Values.zookeeperAntiAffinityPodName }}
release: {{ .Release.Name | quote }}
{{- end }}
{{- end }}
{{- if or (eq .Values.kafkaAntiAffinity "soft") (eq .Values.zookeeperAntiAffinity "soft") }}
preferredDuringSchedulingIgnoredDuringExecution:
{{- if and .Values.kafkaAntiAffinityEnabled (eq .Values.kafkaAntiAffinity "soft") }}
- weight: 1
podAffinityTerm:
topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ include "kafka.name" . | quote }}
release: {{ .Release.Name | quote }}
{{- end }}
{{- if and .Values.zookeeperAntiAffinityEnabled (eq .Values.zookeeperAntiAffinity "soft") }}
- weight: 1
podAffinityTerm:
topologyKey: "kubernetes.io/hostname"
labelSelector:
matchLabels:
app: {{ .Values.zookeeperAntiAffinityPodName }}
{{- end }}
{{- end }}
{{- end }}
{{ toYaml .Values.affinity | indent 8 }}
{{- end }}
containers:
- name: {{ template "kafka.name" . }}-broker
image: "{{ .Values.image }}:{{ .Values.imageTag }}"
incubator/kafka/templates/tests/test_topic_create_consume_produce.yaml
+2 -2
View File
@@ -17,7 +17,7 @@ spec:
# Create a message
MESSAGE="`date -u`" && \
# Produce a test message to the topic
echo $MESSAGE | kafka-console-producer --broker-list {{ template "kafka.fullname" . }}:9092 --topic helm-test-topic-create-consume-produce && \
echo "$MESSAGE" | kafka-console-producer --broker-list {{ template "kafka.fullname" . }}:9092 --topic helm-test-topic-create-consume-produce && \
# Consume a test message from the topic
kafka-console-consumer --bootstrap-server {{ template "kafka.fullname" . }}-headless:9092 --topic helm-test-topic-create-consume-produce --from-beginning --timeout-ms 2000 | grep "$MESSAGE"
kafka-console-consumer --bootstrap-server {{ template "kafka.fullname" . }}-headless:9092 --topic helm-test-topic-create-consume-produce --from-beginning --timeout-ms 2000 --max-messages 1 | grep "$MESSAGE"
restartPolicy: Never
incubator/kafka/values.yaml
+4 -22
View File
@@ -40,28 +40,10 @@ dataDirectory: "/opt/kafka/data"
## This is combined with `dataDirectory` above, to create, by default: /opt/kafka/data/logs
logSubPath: "logs"
## Attempt to prevent Kafka pods from being colocated with eachother.
kafkaAntiAffinityEnabled: true
## Prevent Kafka pods from being colocated on the same node.
## `hard` means the scheduler must satisfy the constraints or the pod fails to schedule
## `soft` means the scheduler makes a best-effort to satisfy the constraint and schedules it where
## it can if the constraints cannot be met.
## See: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature
kafkaAntiAffinity: "soft"
## Attempt to prevent Kafka and Zookeeper pods from being colocated.
zookeeperAntiAffinityEnabled: true
## Prevent Kafka and Zookeeper pods from being colocated.
## `hard` means the scheduler must satisfy the constraints or the pod fails to schedule
## `soft` means the scheduler makes a best-effort to satisfy the constraint and schedules it where
## it can if the constraints cannot be met.
## See: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature
zookeeperAntiAffinity: "soft"
## The zookeeper pod name which K8s will fuzzy-match against to prevent kafka
## pods from being colocated with zookeeper. If you have more than one zookeeper
## installation in your cluster, make sure to use a more specific pod name to
## indicate which zookeeper pods should be filtered.
zookeeperAntiAffinityPodName: "zookeeper"
## Pod scheduling preferences.
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
##
affinity: {}
## Configuration Overrides. Specify any Kafka settings you would like set on the StatefulSet
## here in map format, as defined in the official docs: