Kafka stream data pipeline

docker-compose.yaml

version: "3.2"
services:
  zk:
    image: confluentinc/cp-zookeeper:5.1.0
    hostname: zk
    container_name: zk
    restart: unless-stopped
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_CLIENT_PORT: 2181
      KAFKA_JMX_HOSTNAME: zk
      KAFKA_JMX_PORT: 39999
    volumes:
      - zk-data:/data
      - zk-datalog:/datalog

  kafka:
    image: confluentinc/cp-enterprise-kafka:5.1.0
    hostname: kafka
    container_name: kafka
    restart: unless-stopped
    ports:
      - "9092:9092"
      - "29092:29092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zk:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:9092
      CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zk:2181
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      CONFLUENT_METRICS_ENABLE: 'true'
      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
      KAFKA_JMX_PORT: 49999
    volumes:
      - kafka-data:/var/lib/kafka/data
    depends_on:
      - zk
  
  control-center:
    image: confluentinc/cp-enterprise-control-center:5.1.0
    hostname: control-center
    container_name: control-center
    restart: unless-stopped
    ports:
      - "9021:9021"
    environment:
      CONTROL_CENTER_BOOTSTRAP_SERVERS: 'kafka:9092'
      CONTROL_CENTER_ZOOKEEPER_CONNECT: 'zk:2181'
      CONTROL_CENTER_CONNECT_CLUSTER: 'connect:8083'
      CONTROL_CENTER_KSQL_URL: "http://ksql-server:8088"
      CONTROL_CENTER_KSQL_ADVERTISED_URL: "http://localhost:8088"
      CONTROL_CENTER_REPLICATION_FACTOR: 1
      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
      CONFLUENT_METRICS_TOPIC_REPLICATION: 1
      PORT: 9021
    depends_on:
      - zk
      - kafka
      - schema-registry
      - connect
      - ksql-server

  schema-registry:
    image: confluentinc/cp-schema-registry:5.1.0
    hostname: schema-registry
    container_name: schema-registry
    restart: unless-stopped
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zk:2181
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
      SCHEMA_REGISTRY_JMX_PORT: 9582
      KAFKA_JMX_HOSTNAME: schema-registry
      KAFKA_JMX_PORT: 9582
    depends_on:
      - zk
      - kafka

  schema-registry-ui:
    image: landoop/schema-registry-ui:0.9.5
    hostname: schema-registry-ui
    container_name: schema-registry-ui
    restart: unless-stopped
    ports:
      - "8001:8000"
    environment:
      SCHEMAREGISTRY_URL: http://schema-registry:8081/
      PROXY: "true"
    depends_on:
      - schema-registry

  rest-proxy:
    image: confluentinc/cp-kafka-rest:5.1.0
    hostname: rest-proxy
    container_name: rest-proxy
    restart: unless-stopped
    ports:
      - "8082:8082"
    environment:
      KAFKA_REST_ZOOKEEPER_CONNECT: zk:2181
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
      KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry:8081/
      KAFKA_REST_HOST_NAME: rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: kafka:9092
    depends_on:
      - zk
      - kafka
      - schema-registry

  ksql-server:
    image: confluentinc/cp-ksql-server:5.1.0
    hostname: ksql-server
    container_name: ksql-server
    restart: unless-stopped
    ports:
      - "8088:8088"
    environment:
      KSQL_CONFIG_DIR: "/etc/ksql"
      #KSQL_OPTS: ""
      KSQL_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties"
      KSQL_BOOTSTRAP_SERVERS: "kafka:9092"
      KSQL_HOST_NAME: ksql-server
      KSQL_APPLICATION_ID: "cp-all-in-one"
      KSQL_LISTENERS: "http://0.0.0.0:8088"
      KSQL_CACHE_MAX_BYTES_BUFFERING: 0
      KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
      KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
      KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
    depends_on:
      - kafka
      - schema-registry
  
  ksql-cli:
    image: confluentinc/cp-ksql-cli:5.1.0
    container_name: ksql-cli
    restart: unless-stopped
    depends_on:
      - kafka
      - connect
      - ksql-server
    entrypoint: /bin/sh
    tty: true
  
  datagen-pageviews:
    image: confluentinc/ksql-examples:5.1.0
    hostname: datagen-pageviews
    container_name: datagen-pageviews
    restart: unless-stopped
    command: "bash -c 'echo Waiting for Kafka to be ready... && \
                       cub kafka-ready -b kafka:9092 1 40 && \
                       echo Waiting for Confluent Schema Registry to be ready... && \
                       cub sr-ready schema-registry 8081 40 && \
                       echo Waiting a few seconds for topic creation to finish... && \
                       sleep 11 && \
                       ksql-datagen \
                        bootstrap-server=kafka:9092 \
                        quickstart=pageviews \
                        format=delimited \
                        topic=pageviews \
                        maxInterval=500'"
    environment:
      KSQL_CONFIG_DIR: "/etc/ksql"
      KSQL_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties"
      STREAMS_BOOTSTRAP_SERVERS: kafka:9092
      STREAMS_SCHEMA_REGISTRY_HOST: schema-registry
      STREAMS_SCHEMA_REGISTRY_PORT: 8081
    depends_on:
      - ksql-server
      - kafka
      - schema-registry
      - connect
  
  datagen-users:
    image: confluentinc/ksql-examples:5.1.0
    hostname: datagen-users
    container_name: datagen-users
    restart: unless-stopped
    command: "bash -c 'echo Waiting for Kafka to be ready... && \
                       cub kafka-ready -b kafka:9092 1 40 && \
                       echo Waiting for Confluent Schema Registry to be ready... && \
                       cub sr-ready schema-registry 8081 40 && \
                       echo Waiting a few seconds for topic creation to finish... && \
                       sleep 11 && \
                       ksql-datagen \
                        bootstrap-server=kafka:9092 \
                        quickstart=users \
                        format=json \
                        topic=users \
                        maxInterval=100'"
    environment:
      KSQL_CONFIG_DIR: "/etc/ksql"
      KSQL_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties"
      STREAMS_BOOTSTRAP_SERVERS: kafka:9092
      STREAMS_SCHEMA_REGISTRY_HOST: schema-registry
      STREAMS_SCHEMA_REGISTRY_PORT: 8081
    depends_on:
      - ksql-server
      - kafka
      - schema-registry
      - connect
 
  kafka-topics-ui:
    image: landoop/kafka-topics-ui:0.9.4
    hostname: kafka-topics-ui
    container_name: kafka-topics-ui
    restart: unless-stopped
    ports:
      - "8000:8000"
    environment:
      KAFKA_REST_PROXY_URL: "http://rest-proxy:8082/"
      PROXY: "true"
    depends_on:
      - zk
      - kafka
      - schema-registry
      - rest-proxy

  connect:
    image: confluentinc/cp-kafka-connect:5.1.0
    hostname: connect
    container_name: connect
    restart: unless-stopped
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka:9092"
      CONNECT_REST_ADVERTISED_HOST_NAME: connect
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: compose-connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-5.1.0.jar
      CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
      CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
      CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
      CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-5.0.0.jar
      KAFKA_JMX_HOSTNAME: connect
      KAFKA_JMX_PORT: 9584
    depends_on:
      - zk
      - kafka
      - schema-registry
      - rest-proxy

  kafka-connect-ui:
    image: landoop/kafka-connect-ui:0.9.6
    hostname: kafka-connect-ui
    container_name: kafka-connect-ui
    restart: unless-stopped
    ports:
      - "8003:8000"
    environment:
      CONNECT_URL: "http://connect:8083/"
      PROXY: "true"
    depends_on:
      - connect

  zoonavigator-web:
    image: elkozmon/zoonavigator-web:0.5.1
    hostname: zoonavigator-web
    container_name: zoonavigator-web
    restart: unless-stopped
    ports:
     - "8004:8000"
    environment:
      API_HOST: "zoonavigator-api"
      API_PORT: 9000
      #AUTO_CONNECT_CONNECTION_STRING: ""
      #AUTO_CONNECT_AUTH_INFO: 
    links:
     - zoonavigator-api
    depends_on:
     - zoonavigator-api

  zoonavigator-api:
    image: elkozmon/zoonavigator-api:0.5.1
    hostname: oonavigator-api
    container_name: zoonavigator-api
    restart: unless-stopped
    environment:
      SERVER_HTTP_PORT: 9000
    depends_on:
      - zk
  
  kafka-manager:
    image: solsson/kafka-manager
    container_name: kafka-manager
    restart: unless-stopped
    ports:
     - "8005:80"
    environment:
      ZK_HOSTS: "zk:2181"
    command: -Dhttp.port=80
    depends_on:
      - kafka
  
  lenses:
    image: landoop/lenses:2.2.2
    hostname: lenses
    container_name: lenses
    restart: unless-stopped
    ports:
      - 9991:9991
      - 9102:9102
    environment:
      LENSES_PORT: 9991
      LENSES_KAFKA_BROKERS: "PLAINTEXT://kafka:9092"
      #LICENSE: "$(cat license.json)"
      LENSES_ZOOKEEPER_HOSTS: |
        [
          {url:"zk:2181", jmx:"zk:39999"}
        ]
      LENSES_SCHEMA_REGISTRY_URLS: |
        [
          {url:"http://schema-registry:8081",jmx:"schema-registry:9582"}
        ]
      LENSES_CONNECT_CLUSTERS: |
        [
          {
            name:"data_science",
            urls: [
              {url:"http://connect:8083",jmx:"connect:9584"}
            ],
            statuses:"docker-connect-statuses",
            configs:"docker-connect-configs",
            offsets:"docker-connect-offsets"
          }
        ]
      LENSES_SECURITY_MODE: BASIC
      # Secrets can also be passed as files. Check _examples/
      LENSES_SECURITY_GROUPS: |
        [
          {"name": "adminGroup", "roles": ["admin", "write", "read"]},
          {"name": "readGroup",  "roles": ["read"]}
        ]
      LENSES_SECURITY_USERS: |
        [
          {"username": "admin", "password": "admin", "displayname": "Lenses Admin", "groups": ["adminGroup"]},
          {"username": "read", "password": "read", "displayname": "Read Only", "groups": ["readGroup"]}
        ]
    volumes:
      - lenses-log:/data/log
      - lenses-kafka-state:/data/kafka-streams-state
      - ./license.json:/data/license.json

volumes:
  mi2: {}
  zk-data: {}
  zk-datalog: {} 
  kafka-data: {}
  lenses-log: {}
  lenses-kafka-state: {}