Deploy Kafka with Kraft | Systemd unit

kafka_deploy.md

Deploy Kafka with Kraft | Systemd unit

Create user kafka

adduser kafka
adduser kafka sudo
su - kafka

Description

Config	Value
Kafka path	/opt/kafka
Main config	/opt/kafka/config/kraft/server.properties
Data Store	/var/lib/kafka
Logs	/var/log/kafka
advertised.listeners	PLAINTEXT://<PRIVATE_IPADDR>:9092, PLAINTEXT://localhost:9091
controller.listeners	PLAINTEXT://<PRIVATE_IPADDR>:9093

Download kafka archive

sudo apt update && sudo apt install -y openjdk-17-jdk
sudo wget https://downloads.apache.org/kafka/3.7.2/kafka_2.12-3.7.2.tgz
sudo mkdir /opt/kafka; sudo tar -xf kafka_2.12-3.7.2.tgz -C /opt/kafka --strip-components 1
sudo chown -R kafka:kafka /opt/kafka
sudo mkdir -p /var/lib/kafka && chown -R kafka:kafka /var/lib/kafka
sudo mkdir -p /var/log/kafka && chown -R kafka:kafka /var/log/kafka

Generate a Cluster UUID

export KAFKA_CLUSTER_ID="$(/opt/kafka/bin/kafka-storage.sh random-uuid)"
echo $KAFKA_CLUSTER_ID

Modify Kraft config (optional)

# <node_id>@<ip_address>:<port>
controller.quorum.voters=1@10.0.0.77:9093
listeners=PLAINTEXT_LOCAL://127.0.0.1:9091,CONTROLLER://10.0.0.77:9093,PLAINTEXT://10.0.0.77:9092
listener.security.protocol.map=PLAINTEXT:PLAINTEXT,PLAINTEXT_LOCAL:PLAINTEXT,CONTROLLER:PLAINTEXT
controller.listener.names=CONTROLLER
advertised.listeners=PLAINTEXT_LOCAL://127.0.0.1:9091,PLAINTEXT://10.0.0.77:9092
log.dirs=/var/lib/kafka
cluster.id=DClWFmdXTp2TpJLuv_bF4A
process.roles=broker,controller

queued.max.requests=5000
socket.request.max.bytes=104857600
message.max.bytes=10485760

num.network.threads=4
# Increase if disk/network IO is high
num.io.threads=8
num.replica.fetchers=4

# 1GB per segment
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000

offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1

node.id=1

vi /opt/kafka/config/kraft/server.properties

Modify kafka log

Create folder if not exists

sudo mkdir -p /var/log/kafka
sudo chown -R kafka:kafka /var/log/kafka

Create file

sudo vi /opt/kafka/config/log4j2.yaml

Modify content

Before

Configuration:
  Properties:
    Property:
      # Fallback if the system property is not set
      - name: "kafka.logs.dir"
        value: "."

After

Configuration:
  Properties:
    Property:
      # Fallback if the system property is not set
      - name: "kafka.logs.dir"
        value: "/var/log/kafka"

Format Log Directories

/opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c /opt/kafka/config/kraft/server.properties

kafka-server.service

cat <<EOF > /etc/systemd/system/kafka.service

[Unit]
Description=Apache Kafka server (broker)
Documentation=http://kafka.apache.org/documentation.html
After=network.target remote-fs.target
Requires=network.target remote-fs.target


[Service]
Type=simple
User=kafka
Group=sudo
Environment=JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
Environment="KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:/opt/kafka/config/log4j.properties -Dkafka.logs.dir=/var/log/kafka"
ExecStart=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kraft/server.properties
ExecStop=/opt/kafka/bin/kafka-server-stop.sh

[Install]
WantedBy=multi-user.target
Alias=kafka.service

EOF

Systemctl

systemctl daemon-reload
systemctl enable kafka.service
systemctl start kafka.service
systemctl status kafka.service

Test

# Produce
echo "test-msg" | docker run -i --network=host edenhill/kcat:1.7.1 -P -b localhost:9091 -t test

# Consume
docker run -it --network=host edenhill/kcat:1.7.1   -b localhost:9091  -G test_group test

# List internal topic (if not exists, unavailable to consume message)
/opt/kafka/bin/kafka-topics.sh   --bootstrap-server 127.0.0.1:9091   --describe --topic __consumer_offsets