diff --git a/incubator/kafka/Chart.yaml b/incubator/kafka/Chart.yaml index 0e07790584..159c30fc02 100755 --- a/incubator/kafka/Chart.yaml +++ b/incubator/kafka/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 description: Apache Kafka is publish-subscribe messaging rethought as a distributed commit log. name: kafka -version: 0.14.3 +version: 0.14.4 appVersion: 5.0.1 keywords: - kafka diff --git a/incubator/kafka/README.md b/incubator/kafka/README.md index b041ac9e63..089007b9b5 100644 --- a/incubator/kafka/README.md +++ b/incubator/kafka/README.md @@ -175,9 +175,9 @@ Kafka has a rich ecosystem, with lots of tools. This sections is intended to com - [Schema-registry](https://github.com/kubernetes/charts/tree/master/incubator/schema-registry) - A confluent project that provides a serving layer for your metadata. It provides a RESTful interface for storing and retrieving Avro schemas. -### Connecting to Kafka from outside Kubernetes +## Connecting to Kafka from outside Kubernetes -#### Node Port External Service Type +### NodePort External Service Type Review and optionally override to enable the example text concerned with external access in `values.yaml`. @@ -208,10 +208,123 @@ the a `containerPort` with a number matching its respective `NodePort`. The rang should not actually listen, on all Kafka pods in the StatefulSet. As any given pod will listen only one such port at a time, setting the range at every Kafka pod is a reasonably safe configuration. -#### Load Balancer External Service Type +#### Example values.yml for external service type NodePort +The + lines are with the updated values. +``` + external: +- enabled: false ++ enabled: true + # type can be either NodePort or LoadBalancer + type: NodePort + # annotations: +@@ -170,14 +170,14 @@ configurationOverrides: + ## + ## Setting "advertised.listeners" here appends to "PLAINTEXT://${POD_IP}:9092,", ensure you update the domain + ## If external service type is Nodeport: +- # "advertised.listeners": |- +- # EXTERNAL://kafka.cluster.local:$((31090 + ${KAFKA_BROKER_ID})) ++ "advertised.listeners": |- ++ EXTERNAL://kafka.cluster.local:$((31090 + ${KAFKA_BROKER_ID})) + ## If external service type is LoadBalancer: + # "advertised.listeners": |- + # EXTERNAL://kafka-$((${KAFKA_BROKER_ID})).cluster.local:19092 + ## Uncomment to define the EXTERNAL Listener protocol +- # "listener.security.protocol.map": |- +- # PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT ++ "listener.security.protocol.map": |- ++ PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT + + +$ kafkacat -b kafka.cluster.local:31090 -L +Metadata for all topics (from broker 0: kafka.cluster.local:31090/0): + 3 brokers: + broker 2 at kafka.cluster.local:31092 + broker 1 at kafka.cluster.local:31091 + broker 0 at kafka.cluster.local:31090 + 0 topics: + +$ kafkacat -b kafka.cluster.local:31090 -P -t test1 -p 0 +msg01 from external producer to topic test1 + +$ kafkacat -b kafka.cluster.local:31090 -C -t test1 -p 0 +msg01 from external producer to topic test1 +``` +### LoadBalancer External Service Type The load balancer external service type differs from the node port type by routing to the `external.servicePort` specified in the service for each statefulset container (if `external.distinct` is set). If `external.distinct` is false, `external.servicePort` is unused and will be set to the sum of `external.firstListenerPort` and the replica number. It is important to note that `external.firstListenerPort` does not have to be within the configured node port range for the cluster, however a node port will be allocated. +#### Example values.yml and DNS setup for external service type LoadBalancer +The + lines are with the updated values. +``` + external: +- enabled: false ++ enabled: true + # type can be either NodePort or LoadBalancer +- type: NodePort ++ type: LoadBalancer + # annotations: + # service.beta.kubernetes.io/openstack-internal-load-balancer: "true" + dns: +@@ -138,10 +138,10 @@ external: + # If using external service type LoadBalancer and external dns, set distinct to true below. + # This creates an A record for each statefulset pod/broker. You should then map the + # A record of the broker to the EXTERNAL IP given by the LoadBalancer in your DNS server. +- distinct: false ++ distinct: true + servicePort: 19092 + firstListenerPort: 31090 +- domain: cluster.local ++ domain: example.com + loadBalancerIP: [] + init: + image: "lwolf/kubectl_deployer" +@@ -173,11 +173,11 @@ configurationOverrides: + # "advertised.listeners": |- + # EXTERNAL://kafka.cluster.local:$((31090 + ${KAFKA_BROKER_ID})) + ## If external service type is LoadBalancer: +- # "advertised.listeners": |- +- # EXTERNAL://kafka-$((${KAFKA_BROKER_ID})).cluster.local:19092 ++ "advertised.listeners": |- ++ EXTERNAL://kafka-$((${KAFKA_BROKER_ID})).example.com:19092 + ## Uncomment to define the EXTERNAL Listener protocol +- # "listener.security.protocol.map": |- +- # PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT ++ "listener.security.protocol.map": |- ++ PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT + +$ kubectl -n kafka get svc +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +kafka ClusterIP 10.39.241.217 9092/TCP 2m39s +kafka-0-external LoadBalancer 10.39.242.45 35.200.238.174 19092:30108/TCP 2m39s +kafka-1-external LoadBalancer 10.39.241.90 35.244.44.162 19092:30582/TCP 2m39s +kafka-2-external LoadBalancer 10.39.243.160 35.200.149.80 19092:30539/TCP 2m39s +kafka-headless ClusterIP None 9092/TCP 2m39s +kafka-zookeeper ClusterIP 10.39.249.70 2181/TCP 2m39s +kafka-zookeeper-headless ClusterIP None 2181/TCP,3888/TCP,2888/TCP 2m39s + +DNS A record entries: +kafka-0.example.com A record 35.200.238.174 TTL 60sec +kafka-1.example.com A record 35.244.44.162 TTL 60sec +kafka-2.example.com A record 35.200.149.80 TTL 60sec + +$ ping kafka-0.example.com +PING kafka-0.example.com (35.200.238.174): 56 data bytes + +$ kafkacat -b kafka-0.example.com:19092 -L +Metadata for all topics (from broker 0: kafka-0.example.com:19092/0): + 3 brokers: + broker 2 at kafka-2.example.com:19092 + broker 1 at kafka-1.example.com:19092 + broker 0 at kafka-0.example.com:19092 + 0 topics: + +$ kafkacat -b kafka-0.example.com:19092 -P -t gkeTest -p 0 +msg02 for topic gkeTest + +$ kafkacat -b kafka-0.example.com:19092 -C -t gkeTest -p 0 +msg02 for topic gkeTest +``` + ## Known Limitations * Only supports storage options that have backends for persistent volume claims (tested mostly on AWS)