mirror of
https://github.com/helm/charts.git
synced 2026-08-23 22:37:45 +00:00
More features and fixes (Secrets/ENVs/ServicePort) (#7653)
* [incubator/kafka] fix script and kafka test Signed-off-by: Steffen Görlach <stevie-@users.noreply.github.com> * [incubator/kafka] add feature acls also add acls if specified in topics Signed-off-by: Steffen Görlach <stevie-@users.noreply.github.com> * [incubator/kafka] simple secret mount be able to only mount a secret to a path into container. This supports build in /etc/kafka/secrets mountpoint in confluent docker image Signed-off-by: Steffen Görlach <stevie-@users.noreply.github.com> * [incubator/kafka] more features - be able to only mount a secret to a path into container. -- This supports build in /etc/kafka/secrets mountpoint in confluent docker image - be able to set env variables - be able to disable DNS annotations which may conflict if enabled both - if we have distict external hostnames use service port Signed-off-by: Steffen Görlach <stevie-@users.noreply.github.com> * [incubator/kafka] support to set multiple acl operations Signed-off-by: stevie- <stevie-@users.noreply.github.com> * [incubator/kafka] support to set multiple acl operations Signed-off-by: stevie- <stevie-@users.noreply.github.com> * [incubator/kafka] bump version + minor format fix Signed-off-by: stevie- <stevie-@users.noreply.github.com> * [incubator/kafka] be able to set scrapeTimeout + bump feature version Signed-off-by: stevie- <stevie-@users.noreply.github.com>
This commit is contained in:
Stef Fen
k8s-ci-robot
@@ -2,7 +2,7 @@ apiVersion: v1
|
||||
description: Apache Kafka is publish-subscribe messaging rethought as a distributed
|
||||
commit log.
|
||||
name: kafka
|
||||
version: 0.10.1
|
||||
version: 0.11.0
|
||||
appVersion: 4.1.2
|
||||
keywords:
|
||||
- kafka
|
||||
|
||||
@@ -71,6 +71,8 @@ following configurable parameters:
|
||||
| `headless.targetPort` | Target port to be used for the headless service. This is not a required value. | `nil` |
|
||||
| `headless.port` | Port to be used for the headless service. https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | `9092` |
|
||||
| `external.enabled` | If True, exposes Kafka brokers via NodePort (PLAINTEXT by default) | `false` |
|
||||
| `external.dns.useInternal` | If True, add Annotation for internal DNS service | `false` |
|
||||
| `external.dns.useExternal` | If True, add Annotation for external DNS service | `true` |
|
||||
| `external.servicePort` | TCP port configured at external services (one per pod) to relay from NodePort to the external listener port. | '19092' |
|
||||
| `external.firstListenerPort` | TCP port which is added pod index number to arrive at the port used for NodePort and external listener port. | '31090' |
|
||||
| `external.domain` | Domain in which to advertise Kafka external listeners. | `cluster.local` |
|
||||
@@ -80,6 +82,7 @@ following configurable parameters:
|
||||
| `external.annotations` | Additional annotations for the external service. | `{}` |
|
||||
| `podAnnotations` | Annotation to be added to Kafka pods | `{}` |
|
||||
| `rbac.enabled` | Enable a service account and role for the init container to use in an RBAC enabled cluster | `false` |
|
||||
| `envOverrides` | Add additional Environment Variables in the dictionary format | `{ zookeeper.sasl.enabled: "False" }` |
|
||||
| `configurationOverrides` | `Kafka ` [configuration setting][brokerconfigs] overrides in the dictionary format | `{ offsets.topic.replication.factor: 3 }` |
|
||||
| `secrets` | `{}` | Pass any secrets to the kafka pods. Each secret will be passed as an environment variable by default. The secret can also be mounted to a specific path if required. Environment variable names are generated as: `<secretName>_<secretKey>` (All upper case)|
|
||||
| `additionalPorts` | Additional ports to expose on brokers. Useful when the image exposes metrics (like prometheus, etc.) through a javaagent instead of a sidecar | `{}` |
|
||||
@@ -105,11 +108,13 @@ following configurable parameters:
|
||||
| `prometheus.jmx.image` | JMX Exporter container image | `solsson/kafka-prometheus-jmx-exporter@sha256` |
|
||||
| `prometheus.jmx.imageTag` | JMX Exporter container image tag | `a23062396cd5af1acdf76512632c20ea6be76885dfc20cd9ff40fb23846557e8` |
|
||||
| `prometheus.jmx.interval` | Interval that Prometheus scrapes JMX metrics when using Prometheus Operator | `10s` |
|
||||
| `prometheus.jmx.scrapeTimeout` | Timeout that Prometheus scrapes JMX metrics when using Prometheus Operator | `10s` |
|
||||
| `prometheus.jmx.port` | JMX Exporter Port which exposes metrics in Prometheus format for scraping | `5556` |
|
||||
| `prometheus.kafka.enabled` | Whether or not to create a separate Kafka exporter | `false` |
|
||||
| `prometheus.kafka.image` | Kafka Exporter container image | `danielqsj/kafka-exporter` |
|
||||
| `prometheus.kafka.imageTag` | Kafka Exporter container image tag | `v1.2.0` |
|
||||
| `prometheus.kafka.interval` | Interval that Prometheus scrapes Kafka metrics when using Prometheus Operator | `10s` |
|
||||
| `prometheus.kafka.scrapeTimeout` | Timeout that Prometheus scrapes Kafka metrics when using Prometheus Operator | `10s` |
|
||||
| `prometheus.kafka.port` | Kafka Exporter Port which exposes metrics in Prometheus format for scraping | `9308` |
|
||||
| `prometheus.kafka.resources` | Allows setting resource limits for kafka-exporter pod | `{}` |
|
||||
| `prometheus.operator.enabled` | True if using the Prometheus Operator, False if not | `false` |
|
||||
@@ -203,7 +208,7 @@ such port at a time, setting the range at every Kafka pod is a reasonably safe c
|
||||
|
||||
#### Load Balancer External Service Type
|
||||
|
||||
The load balancer external service type differs from the node port type by routing to the `port` specified in the service for each statefulset container. Because of this `external.servicePort` is unused and will be set to the sum of `external.firstListenerPort` and the replica number. It is important to note that `external.firstListenerPort` does not have to be within the configured node port range for the cluster, however a node port will be allocated.
|
||||
The load balancer external service type differs from the node port type by routing to the `external.servicePort` specified in the service for each statefulset container (if `external.distinct` is set). If `external.distinct` is false, `external.servicePort` is unused and will be set to the sum of `external.firstListenerPort` and the replica number. It is important to note that `external.firstListenerPort` does not have to be within the configured node port range for the cluster, however a node port will be allocated.
|
||||
|
||||
## Known Limitations
|
||||
|
||||
|
||||
@@ -51,12 +51,12 @@ clients to point at the following brokers. Please allow a few minutes for all
|
||||
associated resources to become healthy.
|
||||
{{ $fullName := include "kafka.fullname" . }}
|
||||
{{- $replicas := .Values.replicas | int }}
|
||||
{{- $servicePort := .Values.external.servicePort }}
|
||||
{{- $servicePort := .Values.external.servicePort | int}}
|
||||
{{- $root := . }}
|
||||
{{- range $i, $e := until $replicas }}
|
||||
{{- $externalListenerPort := add $root.Values.external.firstListenerPort $i }}
|
||||
{{- if $root.Values.external.distinct }}
|
||||
{{ printf "%s-%d.%s:%d" $root.Release.Name $i $root.Values.external.domain $externalListenerPort | indent 2 }}
|
||||
{{ printf "%s-%d.%s:%d" $root.Release.Name $i $root.Values.external.domain $servicePort | indent 2 }}
|
||||
{{- else }}
|
||||
{{ printf "%s.%s:%d" $root.Release.Name $root.Values.external.domain $externalListenerPort | indent 2 }}
|
||||
{{- end }}
|
||||
|
||||
@@ -25,7 +25,7 @@ data:
|
||||
sleep 20
|
||||
done
|
||||
echo "Applying runtime configuration using {{ .Values.image }}:{{ .Values.imageTag }}"
|
||||
{{- range $n, $topic := .Values.topics }}
|
||||
{{- range $n, $topic := .Values.topics }}
|
||||
{{- if and $topic.partitions $topic.replicationFactor $topic.reassignPartitions }}
|
||||
cat << EOF > {{ $topic.name }}-increase-replication-factor.json
|
||||
{"version":1, "partitions":[
|
||||
@@ -50,5 +50,12 @@ data:
|
||||
kafka-configs --zookeeper {{ $zk }} --entity-type topics --entity-name {{ $topic.name }} --alter --force --add-config {{ nospace $topic.config }}
|
||||
{{- end }}
|
||||
kafka-configs --zookeeper {{ $zk }} --entity-type topics --entity-name {{ $topic.name }} --describe
|
||||
{{- if $topic.acls }}
|
||||
{{- range $a, $acl := $topic.acls }}
|
||||
{{ if and $acl.user $acl.operations }}
|
||||
kafka-acls --authorizer-properties zookeeper.connect={{ $zk }} --force --add --allow-principal User:{{ $acl.user }}{{- range $operation := $acl.operations }} --operation {{ $operation }} {{- end }} --topic {{ $topic.name }} {{ $topic.extraParams }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
@@ -14,11 +14,19 @@ kind: Service
|
||||
metadata:
|
||||
annotations:
|
||||
{{- if $root.Values.external.distinct }}
|
||||
{{- if $root.Values.external.dns.useInternal }}
|
||||
dns.alpha.kubernetes.io/internal: "{{ $distinctPrefix }}.{{ $root.Values.external.domain }}"
|
||||
{{- end }}
|
||||
{{- if $root.Values.external.dns.useExternal }}
|
||||
external-dns.alpha.kubernetes.io/hostname: "{{ $distinctPrefix }}.{{ $root.Values.external.domain }}"
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
{{- if $root.Values.external.dns.useInternal }}
|
||||
dns.alpha.kubernetes.io/internal: "{{ $dnsPrefix }}.{{ $root.Values.external.domain }}"
|
||||
{{- end }}
|
||||
{{- if $root.Values.external.dns.useExternal }}
|
||||
external-dns.alpha.kubernetes.io/hostname: "{{ $dnsPrefix }}.{{ $root.Values.external.domain }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if $root.Values.external.annotations }}
|
||||
{{ toYaml $root.Values.external.annotations | indent 4 }}
|
||||
@@ -34,12 +42,13 @@ spec:
|
||||
type: {{ $root.Values.external.type }}
|
||||
ports:
|
||||
- name: external-broker
|
||||
{{- if eq $root.Values.external.type "LoadBalancer" }}
|
||||
{{- if and (eq $root.Values.external.type "LoadBalancer") (not $root.Values.external.distinct) }}
|
||||
port: {{ $externalListenerPort }}
|
||||
targetPort: {{ $externalListenerPort }}
|
||||
{{- else }}
|
||||
port: {{ $servicePort }}
|
||||
targetPort: {{ $servicePort }}
|
||||
{{- end }}
|
||||
targetPort: {{ $externalListenerPort }}
|
||||
{{- if eq $root.Values.external.type "NodePort" }}
|
||||
nodePort: {{ $externalListenerPort }}
|
||||
{{- end }}
|
||||
|
||||
@@ -14,6 +14,9 @@ spec:
|
||||
endpoints:
|
||||
- port: jmx-exporter
|
||||
interval: {{ .Values.prometheus.jmx.interval }}
|
||||
{{- if .Values.prometheus.jmx.scrapeTimeout }}
|
||||
scrapeTimeout: {{ .Values.prometheus.jmx.scrapeTimeout }}
|
||||
{{- end }}
|
||||
namespaceSelector:
|
||||
any: true
|
||||
{{ end }}
|
||||
@@ -34,6 +37,9 @@ spec:
|
||||
endpoints:
|
||||
- port: kafka-exporter
|
||||
interval: {{ .Values.prometheus.kafka.interval }}
|
||||
{{- if .Values.prometheus.kafka.scrapeTimeout }}
|
||||
scrapeTimeout: {{ .Values.prometheus.kafka.scrapeTimeout }}
|
||||
{{- end }}
|
||||
namespaceSelector:
|
||||
any: true
|
||||
{{ end }}
|
||||
|
||||
@@ -187,6 +187,10 @@ spec:
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range $key, $value := .Values.envOverrides }}
|
||||
- name: {{ printf "%s" $key | replace "." "_" | upper | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- end }}
|
||||
# This is required because the Downward API does not yet support identification of
|
||||
# pod numbering in statefulsets. Thus, we are required to specify a command which
|
||||
# allows us to extract the pod ID for usage as the Kafka Broker ID.
|
||||
@@ -204,10 +208,16 @@ spec:
|
||||
mountPath: {{ .Values.persistence.mountPath | quote }}
|
||||
{{- range $secret := .Values.secrets }}
|
||||
{{- if $secret.mountPath }}
|
||||
{{- range $key := $secret.keys }}
|
||||
{{- if $secret.keys }}
|
||||
{{- range $key := $secret.keys }}
|
||||
- name: {{ include "kafka.fullname" $ }}-{{ $secret.name }}
|
||||
mountPath: {{ $secret.mountPath }}/{{ $key }}
|
||||
subPath: {{ $key }}
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
- name: {{ include "kafka.fullname" $ }}-{{ $secret.name }}
|
||||
mountPath: {{ $secret.mountPath }}
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -138,7 +138,9 @@ external:
|
||||
type: NodePort
|
||||
# annotations:
|
||||
# service.beta.kubernetes.io/openstack-internal-load-balancer: "true"
|
||||
|
||||
dns:
|
||||
useInternal: false
|
||||
useExternal: true
|
||||
# create an A record for each statefulset pod
|
||||
distinct: false
|
||||
enabled: false
|
||||
@@ -175,6 +177,11 @@ configurationOverrides:
|
||||
# "listener.security.protocol.map": |-
|
||||
# PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
|
||||
|
||||
## set extra ENVs
|
||||
# key: "value"
|
||||
envOverrides: {}
|
||||
|
||||
|
||||
## A collection of additional ports to expose on brokers (formatted as normal containerPort yaml)
|
||||
# Useful when the image exposes metrics (like prometheus, etc.) through a javaagent instead of a sidecar
|
||||
additionalPorts: {}
|
||||
@@ -258,6 +265,9 @@ prometheus:
|
||||
## Interval at which Prometheus scrapes metrics, note: only used by Prometheus Operator
|
||||
interval: 10s
|
||||
|
||||
## Timeout at which Prometheus timeouts scrape run, note: only used by Prometheus Operator
|
||||
scrapeTimeout: 10s
|
||||
|
||||
## Port jmx-exporter exposes Prometheus format metrics to scrape
|
||||
port: 5556
|
||||
|
||||
@@ -282,6 +292,9 @@ prometheus:
|
||||
## Interval at which Prometheus scrapes metrics, note: only used by Prometheus Operator
|
||||
interval: 10s
|
||||
|
||||
## Timeout at which Prometheus timeouts scrape run, note: only used by Prometheus Operator
|
||||
scrapeTimeout: 10s
|
||||
|
||||
## Port kafka-exporter exposes for Prometheus to scrape metrics
|
||||
port: 9308
|
||||
|
||||
@@ -336,6 +349,17 @@ topics: []
|
||||
# replicationFactor: 3
|
||||
# defaultConfig: "segment.bytes,segment.ms"
|
||||
# config: "cleanup.policy=compact,delete.retention.ms=604800000"
|
||||
# - name: myAclTopicPartitions
|
||||
# partitions: 8
|
||||
# acls:
|
||||
# - user: read
|
||||
# operations: [ Read ]
|
||||
# - user: read_and_write
|
||||
# operations:
|
||||
# - Read
|
||||
# - Write
|
||||
# - user: all
|
||||
# operations: [ All ]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Zookeeper:
|
||||
|
||||
Reference in New Issue
Block a user