Docker Apache-Kafka+Zookeeper Cluster建置

Apache Kafka 是一個分散式的訊息佇列框架，是由 LinkedIn 公司使用 Scala 語言開發的系統，被廣泛用來處理高吞吐量與容易水平擴展，目前許多巨量資料運算框架以都有整合 Kafka，諸如：Spark、Cloudera、Apache Storm等，

為保證集群高可用，Zookeeper 集群的節點數最好是奇數，最少有三個節點，所以這裡搭建一個三個節點的集群。

實驗機用三台

ZK01 192.168.0.204

ZK02 192.168.0.206

ZK03 192.168.0.218

----------------------------

ZK01機

zookeeper docker設定檔

mkdir zookeeper

cd zookeeper

vi .run.sh

#!/bin/sh
docker rm -f zookeeper
docker run -d \
        --name zookeeper \
        --restart=always \
        --net=host \
        -p 2181:2181 \
        -p 2888:2888 \
        -p 3888:3888 \
        -e ZOOKEEPER_ID=1 \
        -e ZOOKEEPER_SERVER_1=192.168.0.204 \
        -e ZOOKEEPER_SERVER_2=192.168.0.206 \
        -e ZOOKEEPER_SERVER_3=192.168.0.218 \
        digitalwonderland/zookeeper

chmod +x

./run.sh

kafka docker設定檔

mkdir kafka

cd kafka

vi .run.sh

#/bin/sh
docker rm -f kafka
docker run -d \
        --name kafka \
        --restart=always \
        -p 9092:9092 \
        -e KAFKA_ID=2 \
        -e KAFKA_HOST=192.168.0.206 \
        -e KAFKA_PORT=9092 \
        -e ZOOKEEPER_CONNECT=192.168.0.204:2181,192.168.0.206:2181,192.168.0.218:2181 \
        -e KAFKA_LOG_RETENTION_BYTES=1073741824 \
        -e KAFKA_LOG_retention_hours=24 \
        -e KAFKA_LOG_segment_bytes=1048576 \
        -e KAFKA_LOG_retention_check_interval_ms=3600 \
        -e KAFKA_LOG_cleaner_enable=true \
        jeygeethan/kafka-cluster

chmod +x run.sh

執行./run.sh

#Kafka的集群狀態工具

vi ma-kafka.sh

#!/bin/sh
docker rm -f ma-kafka
docker run -d \
        --name ma-kafka \
        -p 9010:9010 \
        -e ZK_HOSTS=192.168.0.204:2181 \
        -e LISTEN=9010 \
        thomsch98/kafdrop

./ma-kafka.sh

mkdir kafka-manage

vi kafka-manage.sh

#/bin/sh
docker rm -f kafka-manage
docker run -d \
        --name kafka-manage \
        --restart=always \
        -p 9000:9000 \
        -e ZK_HOSTS="192.168.0.204" \
        -e APPLICATION_SECRET=letmein sheepkiller/kafka-manager \
        sheepkiller/kafka-manager

./kafka-manage

檢查docker服務是否都正常運作

ZK02機

zookeeper docker設定檔

mkdir zookeeper

cd zookeeper

vi .run.sh

#!/bin/sh
docker rm -f zookeeper
docker run -d \
        --name zookeeper \
        --restart=always \
        --net=host \
        -p 2181:2181 \
        -p 2888:2888 \
        -p 3888:3888 \
        -e ZOOKEEPER_ID=2 \
        -e ZOOKEEPER_SERVER_1=192.168.0.204 \
        -e ZOOKEEPER_SERVER_2=192.168.0.206 \
        -e ZOOKEEPER_SERVER_3=192.168.0.218 \
        digitalwonderland/zookeeper

chmod +x

./run.sh

kafka docker設定檔

mkdir kafka

cd kafka

vi .run.sh

#/bin/sh
docker rm -f kafka
docker run -d \
        --name kafka \
        --restart=always \
        -p 9092:9092 \
        -e KAFKA_ID=2 \
        -e KAFKA_HOST=192.168.0.206 \
        -e KAFKA_PORT=9092 \
        -e ZOOKEEPER_CONNECT=192.168.0.204:2181,192.168.0.206:2181,192.168.0.218:2181 \
        -e KAFKA_LOG_RETENTION_BYTES=1073741824 \
        -e KAFKA_LOG_retention_hours=24 \
        -e KAFKA_LOG_segment_bytes=1048576 \
        -e KAFKA_LOG_retention_check_interval_ms=3600 \
        -e KAFKA_LOG_cleaner_enable=true \
        jeygeethan/kafka-cluster

chmod +x run.sh

執行./run.sh

ZK03

zookeeper docker設定檔

mkdir zookeeper

cd zookeeper

vi .run.sh

#!/bin/sh
docker rm -f zookeeper
docker run -d \
        --name zookeeper \
        --restart=always \
        --net=host \
        -p 2181:2181 \
        -p 2888:2888 \
        -p 3888:3888 \
        -e ZOOKEEPER_ID=3 \
        -e ZOOKEEPER_SERVER_1=192.168.0.204 \
        -e ZOOKEEPER_SERVER_2=192.168.0.206 \
        -e ZOOKEEPER_SERVER_3=192.168.0.218 \
        digitalwonderland/zookeeper

chmod +x

./run.sh

kafka docker設定檔

mkdir kafka

cd kafka

vi .run.sh

#/bin/sh
docker rm -f kafka
docker run -d \
        --name kafka \
        --restart=always \
        -p 9092:9092 \
        -e KAFKA_ID=3 \
        -e KAFKA_HOST=192.168.0.218 \
        -e KAFKA_PORT=9092 \
        -e ZOOKEEPER_CONNECT=192.168.0.204:2181,192.168.0.206:2181,192.168.0.218:2181 \
        -e KAFKA_LOG_RETENTION_BYTES=1073741824 \
        -e KAFKA_LOG_retention_hours=24 \
        -e KAFKA_LOG_segment_bytes=1048576 \
        -e KAFKA_LOG_retention_check_interval_ms=3600 \
        -e KAFKA_LOG_cleaner_enable=true \
        jeygeethan/kafka-cluster

chmod +x run.sh

執行./run.sh

檢查三台的zookeeper叢集狀態

docker exec -it zookeeper /opt/zookeeper/bin/zkServer.sh status

Mode: leader # Master
Mode: follower # Slave
Mode: standalone # 單機

檢查id
docker exec -it zookeeper cat /data/myid

查看設定檔
docker exec -it zookeeper cat /zookeeper-3.4.13/conf/zoo.cfg

查看 zookeeper 的集群狀態
docker exec -it zookeeper ls -l / | grep zook

開始在ZK01建立的kafka-manege webui

http://192.168.0.204:9000/

建立一個叫log-topic的Topics

http://192.168.0.204:9010/

參考資料：

http://xstarcd.github.io/wiki/Cloud/kafka_Working_Principles.html