diff --git a/incubator/kafka/Chart.yaml b/incubator/kafka/Chart.yaml index 5f8f87338b..acf12d2ae8 100755 --- a/incubator/kafka/Chart.yaml +++ b/incubator/kafka/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 description: Apache Kafka is publish-subscribe messaging rethought as a distributed commit log. name: kafka -version: 0.8.4 +version: 0.8.5 appVersion: 4.1.1 keywords: - kafka diff --git a/incubator/kafka/README.md b/incubator/kafka/README.md index 1355dc8bf0..118fa2e4ac 100644 --- a/incubator/kafka/README.md +++ b/incubator/kafka/README.md @@ -72,6 +72,9 @@ following configurable parameters: | `external.firstListenerPort` | TCP port which is added pod index number to arrive at the port used for NodePort and external listener port. | '31090' | | `external.domain` | Domain in which to advertise Kafka external listeners. | `cluster.local` | | `external.init` | External init container settings. | (see `values.yaml`) | +| `external.type` | Service Type. | `NodePort` | +| `external.distinct` | Distinct DNS entries for each created A record. | `false` | +| `external.annotations` | Additional annotations for the external service. | `{}` | | `rbac.enabled` | Enable a service account and role for the init container to use in an RBAC enabled cluster | `false` | | `configurationOverrides` | `Kafka ` [configuration setting][brokerconfigs] overrides in the dictionary format | `{ offsets.topic.replication.factor: 3 }` | | `additionalPorts` | Additional ports to expose on brokers. Useful when the image exposes metrics (like prometheus, etc.) through a javaagent instead of a sidecar | `{}` | @@ -162,6 +165,8 @@ Kafka has a rich ecosystem, with lots of tools. This sections is intended to com ### Connecting to Kafka from outside Kubernetes +#### Node Port External Service Type + Review and optionally override to enable the example text concerned with external access in `values.yaml`. Once configured, you should be able to reach Kafka via NodePorts, one per replica. In kops where private, @@ -174,6 +179,12 @@ be adjusted to allow non-Kubernetes nodes (e.g. bastion) to access the Kafka ext {{ .Release.Name }}.{{ .Values.external.domain }} ``` +If `external.distinct` is set theses entries will be prefixed with the replica number or broker id. + +``` +{{ .Release.Name }}-.{{ .Values.external.domain }} +``` + Port numbers for external access used at container and NodePort are unique to each container in the StatefulSet. Using the default `external.firstListenerPort` number with a `replicas` value of `3`, the following container and NodePorts will be opened for external access: `31090`, `31091`, `31092`. All of these ports should @@ -185,6 +196,10 @@ the a `containerPort` with a number matching its respective `NodePort`. The rang should not actually listen, on all Kafka pods in the StatefulSet. As any given pod will listen only one such port at a time, setting the range at every Kafka pod is a reasonably safe configuration. +#### Load Balancer External Service Type + +The load balancer external service type differs from the node port type by routing to the `port` specified in the service for each statefulset container. Because of this `external.servicePort` is unused and will be set to the sum of `external.firstListenerPort` and the replica number. It is important to note that `external.firstListenerPort` does not have to be within the configured node port range for the cluster, however a node port will be allocated. + ## Known Limitations * Only supports storage options that have backends for persistent volume claims (tested mostly on AWS) diff --git a/incubator/kafka/templates/NOTES.txt b/incubator/kafka/templates/NOTES.txt index 9c91305084..11eade7b55 100644 --- a/incubator/kafka/templates/NOTES.txt +++ b/incubator/kafka/templates/NOTES.txt @@ -52,11 +52,14 @@ associated resources to become healthy. {{ $fullName := include "kafka.fullname" . }} {{- $replicas := .Values.replicas | int }} {{- $servicePort := .Values.external.servicePort }} - {{- $externalFqdn := printf "%s.%s" .Release.Name .Values.external.domain }} {{- $root := . }} {{- range $i, $e := until $replicas }} {{- $externalListenerPort := add $root.Values.external.firstListenerPort $i }} -{{ printf "%s:%d" $externalFqdn $externalListenerPort | indent 2 }} + {{- if $root.Values.external.distinct }} +{{ printf "%s-%d.%s:%d" $root.Release.Name $i $root.Values.external.domain $externalListenerPort | indent 2 }} + {{- else }} +{{ printf "%s.%s:%d" $root.Release.Name $root.Values.external.domain $externalListenerPort | indent 2 }} + {{- end }} {{- end }} {{- end }} diff --git a/incubator/kafka/templates/service-brokers-external.yaml b/incubator/kafka/templates/service-brokers-external.yaml index ec03cac9ec..e8084f8335 100644 --- a/incubator/kafka/templates/service-brokers-external.yaml +++ b/incubator/kafka/templates/service-brokers-external.yaml @@ -2,17 +2,27 @@ {{- $fullName := include "kafka.fullname" . }} {{- $replicas := .Values.replicas | int }} {{- $servicePort := .Values.external.servicePort }} + {{- $dnsPrefix := printf "%s" .Release.Name }} {{- $root := . }} {{- range $i, $e := until $replicas }} {{- $externalListenerPort := add $root.Values.external.firstListenerPort $i }} {{- $responsiblePod := printf "%s-%d" (printf "%s" $fullName) $i }} + {{- $distinctPrefix := printf "%s-%d" $dnsPrefix $i }} --- apiVersion: v1 kind: Service metadata: annotations: - ## ref: https://github.com/kubernetes/kops/blob/master/dns-controller/pkg/watchers/annotations.go#L21 - dns.alpha.kubernetes.io/internal: "{{ $root.Release.Name }}.{{ $root.Values.external.domain }}" + {{- if $root.Values.external.distinct }} + dns.alpha.kubernetes.io/internal: "{{ $distinctPrefix }}.{{ $root.Values.external.domain }}" + external-dns.alpha.kubernetes.io/hostname: "{{ $distinctPrefix }}.{{ $root.Values.external.domain }}" + {{- else }} + dns.alpha.kubernetes.io/internal: "{{ $dnsPrefix }}.{{ $root.Values.external.domain }}" + external-dns.alpha.kubernetes.io/hostname: "{{ $dnsPrefix }}.{{ $root.Values.external.domain }}" + {{- end }} + {{- if $root.Values.external.annotations }} +{{ toYaml $root.Values.external.annotations | indent 4 }} + {{- end }} name: {{ $root.Release.Name }}-{{ $i }}-external labels: app: {{ include "kafka.name" $root }} @@ -21,12 +31,18 @@ metadata: heritage: {{ $root.Release.Service }} pod: {{ $responsiblePod | quote }} spec: - type: NodePort + type: {{ $root.Values.external.type }} ports: - name: external-broker + {{- if eq $root.Values.external.type "LoadBalancer" }} + port: {{ $externalListenerPort }} + {{- else }} port: {{ $servicePort }} + {{- end }} targetPort: {{ $externalListenerPort }} + {{- if eq $root.Values.external.type "NodePort" }} nodePort: {{ $externalListenerPort }} + {{- end }} protocol: TCP selector: app: {{ include "kafka.name" $root }} diff --git a/incubator/kafka/values.yaml b/incubator/kafka/values.yaml index 1248dadc9f..aba6398dcc 100644 --- a/incubator/kafka/values.yaml +++ b/incubator/kafka/values.yaml @@ -113,6 +113,12 @@ tolerations: [] ## External access. ## external: + type: NodePort + # annotations: + # service.beta.kubernetes.io/openstack-internal-load-balancer: "true" + + # create an A record for each statefulset pod + distinct: false enabled: false servicePort: 19092 firstListenerPort: 31090