- Get the Kubernetes manifests you’ll need for Kafka
git clone https://github.com/Yolean/kubernetes-kafka.git
cd kubernetes-kafka
- Register a couple storage classes in the cluster (note, these probably already exist)
kubectl apply -f ./configure/docker-storageclass-broker.yml
kubectl apply -f ./configure/docker-storageclass-zookeeper.yml
- IMPORTANT: you must edit 00-namespace.yaml to reflect the namespace your team is using. If you do not, you will be sad.
- Now lets start to build our Kubernetes namespace!
kubectl apply -f 00-namespace.yml
kubectl apply -f ./zookeeper
kubectl apply -f ./kafka
kubectl apply -f outside-services/
- Is it there? Find out with the following command
kubectl get all -n your_namespace_name
- OPTIONAL: Yahoo cluster manager
kubectl apply -f ./yahoo-kafka-manager
kubectl port-forward kafka-manager-6977cc4f6f-r58v4 8080:80 -n your_namespace_name
Go to http://localhost:8080 to see yahoo Kafka Cluster manager
Add a cluster on Kafka Manager with this zk host: zookeeper.kafka:2181
- Test it
kubectl exec -it -n your_namespace_name kafka-0 /bin/bash
unset JMX_PORT
bin/kafka-topics.sh -create --zookeeper zookeeper.kafka:2181 --replication-factor 3 --partitions 1 --topic dsetest
- generate some messages
for x in {1..100}; do echo $x; sleep 2; done | bin/kafka-console-producer.sh --topic dsetest --broker-list localhost:9092
- view the messages in this topic
bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic dsetest --from-beginning