diff --git a/incubator/kafka/Chart.yaml b/incubator/kafka/Chart.yaml index a0366c281f..17ce06952c 100755 --- a/incubator/kafka/Chart.yaml +++ b/incubator/kafka/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 description: Apache Kafka is publish-subscribe messaging rethought as a distributed commit log. name: kafka -version: 0.10.1 +version: 0.11.0 appVersion: 4.1.2 keywords: - kafka diff --git a/incubator/kafka/README.md b/incubator/kafka/README.md index e3d10b99f2..7c875bd6ef 100644 --- a/incubator/kafka/README.md +++ b/incubator/kafka/README.md @@ -71,6 +71,8 @@ following configurable parameters: | `headless.targetPort` | Target port to be used for the headless service. This is not a required value. | `nil` | | `headless.port` | Port to be used for the headless service. https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | `9092` | | `external.enabled` | If True, exposes Kafka brokers via NodePort (PLAINTEXT by default) | `false` | +| `external.dns.useInternal` | If True, add Annotation for internal DNS service | `false` | +| `external.dns.useExternal` | If True, add Annotation for external DNS service | `true` | | `external.servicePort` | TCP port configured at external services (one per pod) to relay from NodePort to the external listener port. | '19092' | | `external.firstListenerPort` | TCP port which is added pod index number to arrive at the port used for NodePort and external listener port. | '31090' | | `external.domain` | Domain in which to advertise Kafka external listeners. | `cluster.local` | @@ -80,6 +82,7 @@ following configurable parameters: | `external.annotations` | Additional annotations for the external service. | `{}` | | `podAnnotations` | Annotation to be added to Kafka pods | `{}` | | `rbac.enabled` | Enable a service account and role for the init container to use in an RBAC enabled cluster | `false` | +| `envOverrides` | Add additional Environment Variables in the dictionary format | `{ zookeeper.sasl.enabled: "False" }` | | `configurationOverrides` | `Kafka ` [configuration setting][brokerconfigs] overrides in the dictionary format | `{ offsets.topic.replication.factor: 3 }` | | `secrets` | `{}` | Pass any secrets to the kafka pods. Each secret will be passed as an environment variable by default. The secret can also be mounted to a specific path if required. Environment variable names are generated as: `_` (All upper case)| | `additionalPorts` | Additional ports to expose on brokers. Useful when the image exposes metrics (like prometheus, etc.) through a javaagent instead of a sidecar | `{}` | @@ -105,11 +108,13 @@ following configurable parameters: | `prometheus.jmx.image` | JMX Exporter container image | `solsson/kafka-prometheus-jmx-exporter@sha256` | | `prometheus.jmx.imageTag` | JMX Exporter container image tag | `a23062396cd5af1acdf76512632c20ea6be76885dfc20cd9ff40fb23846557e8` | | `prometheus.jmx.interval` | Interval that Prometheus scrapes JMX metrics when using Prometheus Operator | `10s` | +| `prometheus.jmx.scrapeTimeout` | Timeout that Prometheus scrapes JMX metrics when using Prometheus Operator | `10s` | | `prometheus.jmx.port` | JMX Exporter Port which exposes metrics in Prometheus format for scraping | `5556` | | `prometheus.kafka.enabled` | Whether or not to create a separate Kafka exporter | `false` | | `prometheus.kafka.image` | Kafka Exporter container image | `danielqsj/kafka-exporter` | | `prometheus.kafka.imageTag` | Kafka Exporter container image tag | `v1.2.0` | | `prometheus.kafka.interval` | Interval that Prometheus scrapes Kafka metrics when using Prometheus Operator | `10s` | +| `prometheus.kafka.scrapeTimeout` | Timeout that Prometheus scrapes Kafka metrics when using Prometheus Operator | `10s` | | `prometheus.kafka.port` | Kafka Exporter Port which exposes metrics in Prometheus format for scraping | `9308` | | `prometheus.kafka.resources` | Allows setting resource limits for kafka-exporter pod | `{}` | | `prometheus.operator.enabled` | True if using the Prometheus Operator, False if not | `false` | @@ -203,7 +208,7 @@ such port at a time, setting the range at every Kafka pod is a reasonably safe c #### Load Balancer External Service Type -The load balancer external service type differs from the node port type by routing to the `port` specified in the service for each statefulset container. Because of this `external.servicePort` is unused and will be set to the sum of `external.firstListenerPort` and the replica number. It is important to note that `external.firstListenerPort` does not have to be within the configured node port range for the cluster, however a node port will be allocated. +The load balancer external service type differs from the node port type by routing to the `external.servicePort` specified in the service for each statefulset container (if `external.distinct` is set). If `external.distinct` is false, `external.servicePort` is unused and will be set to the sum of `external.firstListenerPort` and the replica number. It is important to note that `external.firstListenerPort` does not have to be within the configured node port range for the cluster, however a node port will be allocated. ## Known Limitations diff --git a/incubator/kafka/templates/NOTES.txt b/incubator/kafka/templates/NOTES.txt index 11eade7b55..5e0d555115 100644 --- a/incubator/kafka/templates/NOTES.txt +++ b/incubator/kafka/templates/NOTES.txt @@ -51,12 +51,12 @@ clients to point at the following brokers. Please allow a few minutes for all associated resources to become healthy. {{ $fullName := include "kafka.fullname" . }} {{- $replicas := .Values.replicas | int }} - {{- $servicePort := .Values.external.servicePort }} + {{- $servicePort := .Values.external.servicePort | int}} {{- $root := . }} {{- range $i, $e := until $replicas }} {{- $externalListenerPort := add $root.Values.external.firstListenerPort $i }} {{- if $root.Values.external.distinct }} -{{ printf "%s-%d.%s:%d" $root.Release.Name $i $root.Values.external.domain $externalListenerPort | indent 2 }} +{{ printf "%s-%d.%s:%d" $root.Release.Name $i $root.Values.external.domain $servicePort | indent 2 }} {{- else }} {{ printf "%s.%s:%d" $root.Release.Name $root.Values.external.domain $externalListenerPort | indent 2 }} {{- end }} diff --git a/incubator/kafka/templates/configmap-config.yaml b/incubator/kafka/templates/configmap-config.yaml index 062041c26e..9af770235c 100644 --- a/incubator/kafka/templates/configmap-config.yaml +++ b/incubator/kafka/templates/configmap-config.yaml @@ -25,7 +25,7 @@ data: sleep 20 done echo "Applying runtime configuration using {{ .Values.image }}:{{ .Values.imageTag }}" - {{- range $n, $topic := .Values.topics }} + {{- range $n, $topic := .Values.topics }} {{- if and $topic.partitions $topic.replicationFactor $topic.reassignPartitions }} cat << EOF > {{ $topic.name }}-increase-replication-factor.json {"version":1, "partitions":[ @@ -50,5 +50,12 @@ data: kafka-configs --zookeeper {{ $zk }} --entity-type topics --entity-name {{ $topic.name }} --alter --force --add-config {{ nospace $topic.config }} {{- end }} kafka-configs --zookeeper {{ $zk }} --entity-type topics --entity-name {{ $topic.name }} --describe + {{- if $topic.acls }} + {{- range $a, $acl := $topic.acls }} + {{ if and $acl.user $acl.operations }} + kafka-acls --authorizer-properties zookeeper.connect={{ $zk }} --force --add --allow-principal User:{{ $acl.user }}{{- range $operation := $acl.operations }} --operation {{ $operation }} {{- end }} --topic {{ $topic.name }} {{ $topic.extraParams }} + {{- end }} + {{- end }} {{- end }} + {{- end }} {{- end -}} diff --git a/incubator/kafka/templates/service-brokers-external.yaml b/incubator/kafka/templates/service-brokers-external.yaml index e8084f8335..829e35a608 100644 --- a/incubator/kafka/templates/service-brokers-external.yaml +++ b/incubator/kafka/templates/service-brokers-external.yaml @@ -14,11 +14,19 @@ kind: Service metadata: annotations: {{- if $root.Values.external.distinct }} + {{- if $root.Values.external.dns.useInternal }} dns.alpha.kubernetes.io/internal: "{{ $distinctPrefix }}.{{ $root.Values.external.domain }}" + {{- end }} + {{- if $root.Values.external.dns.useExternal }} external-dns.alpha.kubernetes.io/hostname: "{{ $distinctPrefix }}.{{ $root.Values.external.domain }}" + {{- end }} {{- else }} + {{- if $root.Values.external.dns.useInternal }} dns.alpha.kubernetes.io/internal: "{{ $dnsPrefix }}.{{ $root.Values.external.domain }}" + {{- end }} + {{- if $root.Values.external.dns.useExternal }} external-dns.alpha.kubernetes.io/hostname: "{{ $dnsPrefix }}.{{ $root.Values.external.domain }}" + {{- end }} {{- end }} {{- if $root.Values.external.annotations }} {{ toYaml $root.Values.external.annotations | indent 4 }} @@ -34,12 +42,13 @@ spec: type: {{ $root.Values.external.type }} ports: - name: external-broker - {{- if eq $root.Values.external.type "LoadBalancer" }} + {{- if and (eq $root.Values.external.type "LoadBalancer") (not $root.Values.external.distinct) }} port: {{ $externalListenerPort }} + targetPort: {{ $externalListenerPort }} {{- else }} port: {{ $servicePort }} + targetPort: {{ $servicePort }} {{- end }} - targetPort: {{ $externalListenerPort }} {{- if eq $root.Values.external.type "NodePort" }} nodePort: {{ $externalListenerPort }} {{- end }} diff --git a/incubator/kafka/templates/servicemonitors.yaml b/incubator/kafka/templates/servicemonitors.yaml index 92eb1252e8..2bff00c7df 100644 --- a/incubator/kafka/templates/servicemonitors.yaml +++ b/incubator/kafka/templates/servicemonitors.yaml @@ -14,6 +14,9 @@ spec: endpoints: - port: jmx-exporter interval: {{ .Values.prometheus.jmx.interval }} + {{- if .Values.prometheus.jmx.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.jmx.scrapeTimeout }} + {{- end }} namespaceSelector: any: true {{ end }} @@ -34,6 +37,9 @@ spec: endpoints: - port: kafka-exporter interval: {{ .Values.prometheus.kafka.interval }} + {{- if .Values.prometheus.kafka.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.kafka.scrapeTimeout }} + {{- end }} namespaceSelector: any: true {{ end }} diff --git a/incubator/kafka/templates/statefulset.yaml b/incubator/kafka/templates/statefulset.yaml index acbdc060c3..0ac8492d11 100644 --- a/incubator/kafka/templates/statefulset.yaml +++ b/incubator/kafka/templates/statefulset.yaml @@ -187,6 +187,10 @@ spec: {{- end }} {{- end }} {{- end }} + {{- range $key, $value := .Values.envOverrides }} + - name: {{ printf "%s" $key | replace "." "_" | upper | quote }} + value: {{ $value | quote }} + {{- end }} # This is required because the Downward API does not yet support identification of # pod numbering in statefulsets. Thus, we are required to specify a command which # allows us to extract the pod ID for usage as the Kafka Broker ID. @@ -204,10 +208,16 @@ spec: mountPath: {{ .Values.persistence.mountPath | quote }} {{- range $secret := .Values.secrets }} {{- if $secret.mountPath }} - {{- range $key := $secret.keys }} + {{- if $secret.keys }} + {{- range $key := $secret.keys }} - name: {{ include "kafka.fullname" $ }}-{{ $secret.name }} mountPath: {{ $secret.mountPath }}/{{ $key }} subPath: {{ $key }} + readOnly: true + {{- end }} + {{- else }} + - name: {{ include "kafka.fullname" $ }}-{{ $secret.name }} + mountPath: {{ $secret.mountPath }} readOnly: true {{- end }} {{- end }} diff --git a/incubator/kafka/values.yaml b/incubator/kafka/values.yaml index 6c56e45de6..9eec86ef99 100644 --- a/incubator/kafka/values.yaml +++ b/incubator/kafka/values.yaml @@ -138,7 +138,9 @@ external: type: NodePort # annotations: # service.beta.kubernetes.io/openstack-internal-load-balancer: "true" - + dns: + useInternal: false + useExternal: true # create an A record for each statefulset pod distinct: false enabled: false @@ -175,6 +177,11 @@ configurationOverrides: # "listener.security.protocol.map": |- # PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT +## set extra ENVs +# key: "value" +envOverrides: {} + + ## A collection of additional ports to expose on brokers (formatted as normal containerPort yaml) # Useful when the image exposes metrics (like prometheus, etc.) through a javaagent instead of a sidecar additionalPorts: {} @@ -258,6 +265,9 @@ prometheus: ## Interval at which Prometheus scrapes metrics, note: only used by Prometheus Operator interval: 10s + ## Timeout at which Prometheus timeouts scrape run, note: only used by Prometheus Operator + scrapeTimeout: 10s + ## Port jmx-exporter exposes Prometheus format metrics to scrape port: 5556 @@ -282,6 +292,9 @@ prometheus: ## Interval at which Prometheus scrapes metrics, note: only used by Prometheus Operator interval: 10s + ## Timeout at which Prometheus timeouts scrape run, note: only used by Prometheus Operator + scrapeTimeout: 10s + ## Port kafka-exporter exposes for Prometheus to scrape metrics port: 9308 @@ -336,6 +349,17 @@ topics: [] # replicationFactor: 3 # defaultConfig: "segment.bytes,segment.ms" # config: "cleanup.policy=compact,delete.retention.ms=604800000" + # - name: myAclTopicPartitions + # partitions: 8 + # acls: + # - user: read + # operations: [ Read ] + # - user: read_and_write + # operations: + # - Read + # - Write + # - user: all + # operations: [ All ] # ------------------------------------------------------------------------------ # Zookeeper: