docker-compose-kafka.yml

version: '3'

services:
  zookeeper:
    image: confluent/zookeeper:latest
    ports:
      - "2181:2181"
    # volumes:
    #   - ./config/kafka/zk.properties:/etc/kafka/zookeeper.properties

  kafka:
    image: confluent/kafka:0.10.0.0-cp1
    environment:
      #- KAFKA_ADVERTISED_HOST_NAME=
      - KAFKA_BROKER_ID=1
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper
    ports:
      - 9092:9092
    volumes:
      - ./config/kafka/server.properties:/etc/kafka/server.properties
    links:
      - zookeeper
    depends_on:
      - zookeeper

  schema-registry:
    image: confluent/schema-registry:latest
    ports:
      - 8081:8081
    volumes:
      - ./config/kafka/schema-registry.properties:/etc/schema-registry/schema-registry.properties
    links:
      - zookeeper
      - kafka
    depends_on:
      - zookeeper
      - kafka

  rest-proxy:
    image: confluent/rest-proxy:latest
    ports:
      - 8082:8082
    volumes:
      - ./config/kafka/kafka-rest.properties:/etc/kafka-rest/kafka-rest.properties
    links:
      - zookeeper
      - kafka
      - schema-registry
    depends_on:
      - zookeeper
      - kafka
      - schema-registry

  kafka-topics-ui:
    image: landoop/kafka-topics-ui
    environment:
      # yes localhost as its a client side app outsite of the docker network on your host
      - KAFKA_REST_PROXY_URL=http://rest-proxy:8082
      - PROXY=true
    ports:
      - 8000:8000
    links:
      - rest-proxy
    depends_on:
      - rest-proxy

  kafdrop:
    image: thomsch98/kafdrop
    environment:
      # yes localhost as its a client side app outsite of the docker network on your host
      - ZK_HOSTS=zookeeper:2181
      - LISTEN=9000
    ports:
      - 8001:9000
    links:
      - zookeeper
    depends_on:
      - zookeeper

  cli:
    image: taion809/kafka-cli:0.10.2.0
    command: kafka-topics.sh --list --zookeeper zookeeper:2181
    stdin_open: true
    tty: true
    links:
      - zookeeper
    depends_on:
      - zookeeper

Kafka

Running, this will take time to download and startup

docker-compose -f docker-compose-kafka.yml up

You can then access various ui

• http://localhost:8001 # kafdrop - simple admin
• http://localhost:8000 # kafka-topics

list topics

More commands: https://kafka.apache.org/quickstart

# list topics
docker-compose -f docker-compose-kafka.yml run --rm cli kafka-topics.sh --list --zookeeper zookeeper:2181

# create a topic
docker-compose -f docker-compose-kafka.yml run --rm cli kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 1 --topic obb-test

# send data to kafka
docker-compose -f docker-compose-kafka.yml run --rm cli kafka-console-producer.sh --broker-list zookeeper:9092 --topic obb-test


#
# produce and consume messages
# open 2 terminal windows
#

# producer
docker-compose -f docker-compose-kafka.yml run --rm cli kafka-console-producer.sh --broker-list kafka:9092 --topic obb-test
# type some stuff now once it loads up


#consumer - should start seeing what you type
docker-compose -f docker-compose-kafka.yml run --rm cli kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic obb-test