docker-compose_updated.yml

version: '3.8'
services:
  # Elasticsearch service
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.7.1
    hostname: elasticsearch
    container_name: elasticsearchCS
    environment:
      - discovery.type=single-node # Run Elasticsearch as a single-node cluster
      - xpack.security.enabled=false # Disable X-Pack security features
    ports:
      - '9200:9200' # Expose the HTTP API port
    volumes:
      - es_data:/usr/share/elasticsearch/data
    networks:
      - elk_nifi_network

  # Kibana service
  kibana:
    image: docker.elastic.co/kibana/kibana:8.7.1
    hostname: kibana
    container_name: kibanaCS
    environment:
      - ELASTICSEARCH_USERNAME=cybersift
      - ELASTICSEARCH_PASSWORD=changeme
      - xpack.security.enabled=false # Disable X-Pack security features
      - elasticsearch.hosts=http://elasticsearch:9200 # Connect to the Elasticsearch service
    ports:
      - '5601:5601' # Expose the Kibana web interface port
    volumes:
      - kibana_config:/usr/share/kibana/config
    networks:
      - elk_nifi_network
    depends_on:
      - elasticsearch # Make sure Elasticsearch starts first

  # Apache NiFi service
  nifi:
    image: apache/nifi:1.19.0
    container_name: nifiCS
    restart: always # Restart the container if it crashes or stops
    ports:
      - '8080:8080' # Expose the NiFi web interface port
      - '514:514/udp' # Expose the syslog UDP port
      - '9999:9999'
      - '8081:8081'
    volumes:
      - nifi_conf:/opt/nifi/nifi-current/conf # Persist NiFi configuration
      - nifi_flowfile:/opt/nifi/nifi-current/flowfile_repository # Persist NiFi flowfile repository
    environment:
      NIFI_WEB_HTTP_HOST: '0.0.0.0' # Bind the NiFi web interface to all available network interfaces
      NIFI_WEB_HTTP_PORT: 8080 # Set the NiFi web interface port
    networks:
      - elk_nifi_network

  # Filebeat service
  filebeat:
    image: docker.elastic.co/beats/filebeat:8.7.1
    user: root
    container_name: filebeatCS
    volumes:
      - /var/log/messages:/var/log/messages:ro # Mount the local log file as read-only
      - filebeat_data:/usr/share/filebeat/data # Persist Filebeat data
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml # Add Filebeat configuration file
    networks:
      - elk_nifi_network
    depends_on:
      - nifi # Make sure NiFi starts first
      - logstash

  # Logstash service
  logstash:
    image: docker.elastic.co/logstash/logstash:8.7.1
    container_name: logstashCS
    ports:
      - '5044:5044'
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
    environment:
      - 'ELASTICSEARCH_HOSTS=http://elasticsearch:9200'
      - 'ELASTICSEARCH_USERNAME=cybersift'
      - 'ELASTICSEARCH_PASSWORD=changeme'
      - 'XPACK_MONITORING_ENABLED=false'
      - 'XPACK_SECURITY_ENABLED=false'
    networks:
      - elk_nifi_network
    depends_on:
      - elasticsearch

# Custom network for communication between services
networks:
  elk_nifi_network:
    driver: bridge

# Persistent volumes for Elasticsearch, Kibana, and NiFi
volumes:
  es_data:
  kibana_config:
  nifi_conf:
  nifi_flowfile:
  filebeat_data: