Docker zookeeper, kafka cluster 설정

북마크 추가

여기서는 zookeeper 3대와 kafka 3대를 동일 서버내에서 띄우도록 하겠습니다.

1. docker-compose.yaml 작성

$ sudo vi docker-compose.yaml

version: '3.6'
services:
  zookeeper-1:
    image: confluentinc/cp-zookeeper:7.2.1
    container_name: zookeeper-1
    hostname: zookeeper-1
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zookeeper-1:12888:13888;zookeeper-2:22888:23888;zookeeper-3:32888:33888
      ZOOKEEPER_CLIENT_PORT: 2181
     ZOOKEEPER_TICK_TIME: 2000
    ports:
      - 2181:2181
    volumes:
      - ./zk-data/vol1:/var/lib/zookeeper/data
      - ./zk-txn-logs/vol1:/var/lib/zookeeper/logs

  zookeeper-2:
    image: confluentinc/cp-zookeeper:7.2.1
    container_name: zookeeper-2
    hostname: zookeeper-2
    environment:
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_SERVERS: zookeeper-1:12888:13888;zookeeper-2:22888:23888;zookeeper-3:32888:33888
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    ports:
      - 2182:2181
    volumes:
      - ./zk-data/vol2:/var/lib/zookeeper/data
      - ./zk-txn-logs/vol2:/var/lib/zookeeper/logs

  zookeeper-3:
    image: confluentinc/cp-zookeeper:7.2.1
    container_name: zookeeper-3
    hostname: zookeeper-3
    environment:
       ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_SERVERS: zookeeper-1:12888:13888;zookeeper-2:22888:23888;zookeeper-3:32888:33888
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    ports:
      - 2183:2181
    volumes:
      - ./zk-data/vol3:/var/lib/zookeeper/data
      - ./zk-txn-logs/vol3:/var/lib/zookeeper/logs

  kafka-1:
    image: confluentinc/cp-kafka:7.2.1
    container_name: kafka-1
    hostname: kafka-1
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    ports:
      - 19092:19092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092,PLAINTEXT_HOST://{공인아이피}:19092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_ADVERTISED_HOST_NAME: {공인아이피}
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
    volumes:
      - ./kafka-data/vol1:/var/lib/kafka/data
  kafka-2:
    image: confluentinc/cp-kafka:7.2.1
    container_name: kafka-2
    hostname: kafka-2
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    ports:
      - 29092:29092
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092,PLAINTEXT_HOST://{공인아이피}:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_ADVERTISED_HOST_NAME: {공인아이피}
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
    volumes:
      - ./kafka-data/vol2:/var/lib/kafka/data
  kafka-3:
    image: confluentinc/cp-kafka:7.2.1
    container_name: kafka-3
    hostname: kafka-3
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    ports:
      - 39092:39092
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9092,PLAINTEXT_HOST://{공인아이피}:39092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_ADVERTISED_HOST_NAME: {공인아이피}
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
    volumes:
      - ./kafka-data/vol3:/var/lib/kafka/data

- docker image는 confluentinc 의 cp-zookeeper와 cp-kafka를 사용하였고, 7.2.1 버전 입니다.

위 파일을 만들고 최초 실행하게 되면 이미지를 다운 받습니다.

- zookeeper는 외부에서 접속시 2181을 통해 접속 가능합니다.

- kafka의 경우는 19092, 29092, 39092 포트로 구성 하였습니다.

- zookeeper data, log / kafak data는 docker 재시작시 보존을 위해 docker image 외부 volumn 에 저장합니다.

docker-compose.yaml 파일을 생성한 동일 경로에 폴더를 생성합니다.

zookeeper : zk-data/vol1~3

zookeeper log : zk-txn-logs/vol1~3

kafka : kafka-data/vol1~3

3. 도커 실행 및 확인