Setup Confluent Platform Kafka on OpenShift/k8s

confluent-kafka.json

{
  "kind": "Template",
  "apiVersion": "template.openshift.io/v1",
  "metadata": { "name": "lebron-confluent-kafka" },
  "parameters": [],
  "objects": [
    {
      "kind": "DeploymentConfig",
      "apiVersion": "apps.openshift.io/v1",
      "metadata": {
        "name": "lebron-confluent-kafka",
        "labels": { "app": "lebron-confluent-kafka" }
      },
      "spec": {
        "triggers": [{ "type": "ConfigChange" }],
        "replicas": 1,
        "test": false,
        "selector": {
          "deploymentconfig": "lebron-confluent-kafka"
        },
        "template": {
          "metadata": {
            "labels": {
              "deploymentconfig": "lebron-confluent-kafka"
            }
          },
          "spec": {
            "hostname": "lebron-confluent-kafka",
            "containers": [
              {
                "imagePullPolicy": "Always",
                "name": "broker",
                "image": "artifactory.webhostbox.net:9011/confluentinc/cp-server:latest",
                "ports": [{ "containerPort": 9092 }],
                "env": [
                  { "name": "CONFLUENT_METRICS_ENABLE", "value": "true" },
                  {
                    "name": "CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS",
                    "value": "localhost:29092"
                  },
                  {
                    "name": "CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS",
                    "value": "1"
                  },
                  {
                    "name": "CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT",
                    "value": "localhost:2181"
                  },
                  {
                    "name": "CONFLUENT_SUPPORT_CUSTOMER_ID",
                    "value": "anonymous"
                  },
                  {
                    "name": "KAFKA_ADVERTISED_LISTENERS",
                    "value": "PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092"
                  },
                  { "name": "KAFKA_BROKER_ID", "value": "1" },
                  {
                    "name": "KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR",
                    "value": "1"
                  },
                  {
                    "name": "KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS",
                    "value": "0"
                  },
                  {
                    "name": "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP",
                    "value": "PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT"
                  },
                  {
                    "name": "KAFKA_METRIC_REPORTERS",
                    "value": "io.confluent.metrics.reporter.ConfluentMetricsReporter"
                  },
                  {
                    "name": "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR",
                    "value": "1"
                  },
                  {
                    "name": "KAFKA_ZOOKEEPER_CONNECT",
                    "value": "localhost:2181"
                  },
                  {
                    "name": "KAFKA_HEAP_OPTS",
                    "value": "-Xms16G -Xmx16G"
                  }
                ],
                "volumeMounts": [
                  { "mountPath": "/var/lib/kafka/data", "name": "kafka" },
                  { "mountPath": "/etc/kafka/secrets", "name": "kafka" }
                ]
              },
              {
                "imagePullPolicy": "Always",
                "name": "zookeeper",
                "image": "artifactory.webhostbox.net:9011/confluentinc/cp-zookeeper:latest",
                "ports": [{ "containerPort": 2181 }],
                "env": [
                  { "name": "ZOOKEEPER_CLIENT_PORT", "value": "2181" },
                  { "name": "ZOOKEEPER_TICK_TIME", "value": "2000" }
                ],
                "volumeMounts": [
                  { "mountPath": "/var/lib/zookeeper/data", "name": "kafka" },
                  { "mountPath": "/etc/zookeeper/secrets", "name": "kafka" },
                  { "mountPath": "/var/lib/zookeeper/log", "name": "kafka" }
                ]
              },
              {
                "imagePullPolicy": "Always",
                "name": "schema-registry",
                "image": "artifactory.webhostbox.net:9011/confluentinc/cp-schema-registry:latest",
                "ports": [{ "containerPort": 8081 }],
                "env": [
                  {
                    "name": "SCHEMA_REGISTRY_HOST_NAME",
                    "value": "schema-registry"
                  },
                  {
                    "name": "SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL",
                    "value": "localhost:2181"
                  }
                ],
                "volumeMounts": [
                  {
                    "mountPath": "/etc/schema-registry/secrets",
                    "name": "kafka"
                  }
                ],
                "resources": {}
              },
              {
                "imagePullPolicy": "Always",
                "name": "rest-proxy",
                "image": "artifactory.webhostbox.net:9011/confluentinc/cp-kafka-rest:latest",
                "ports": [{ "containerPort": 8082 }],
                "env": [
                  {
                    "name": "KAFKA_REST_BOOTSTRAP_SERVERS",
                    "value": "localhost:29092"
                  },
                  { "name": "KAFKA_REST_HOST_NAME", "value": "rest-proxy" },
                  {
                    "name": "KAFKA_REST_LISTENERS",
                    "value": "http://0.0.0.0:8082"
                  },
                  {
                    "name": "KAFKA_REST_SCHEMA_REGISTRY_URL",
                    "value": "http://localhost:8081"
                  }
                ],
                "resources": {}
              },
              {
                "imagePullPolicy": "Always",
                "name": "connect",
                "image": "artifactory.webhostbox.net:9011/confluentinc/cp-server-connect:latest",
                "ports": [{ "containerPort": 8083 }],
                "env": [
                  {
                    "name": "CLASSPATH",
                    "value": "/usr/share/java/monitoring-interceptors/monitoring-interceptors-5.4.1.jar"
                  },
                  {
                    "name": "CONNECT_BOOTSTRAP_SERVERS",
                    "value": "localhost:29092"
                  },
                  {
                    "name": "CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR",
                    "value": "1"
                  },
                  {
                    "name": "CONNECT_CONFIG_STORAGE_TOPIC",
                    "value": "docker-connect-configs"
                  },
                  {
                    "name": "CONNECT_CONSUMER_INTERCEPTOR_CLASSES",
                    "value": "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
                  },
                  {
                    "name": "CONNECT_GROUP_ID",
                    "value": "compose-connect-group"
                  },
                  {
                    "name": "CONNECT_INTERNAL_KEY_CONVERTER",
                    "value": "org.apache.kafka.connect.json.JsonConverter"
                  },
                  {
                    "name": "CONNECT_INTERNAL_VALUE_CONVERTER",
                    "value": "org.apache.kafka.connect.json.JsonConverter"
                  },
                  {
                    "name": "CONNECT_KEY_CONVERTER",
                    "value": "org.apache.kafka.connect.storage.StringConverter"
                  },
                  {
                    "name": "CONNECT_LOG4J_LOGGERS",
                    "value": "org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR"
                  },
                  {
                    "name": "CONNECT_OFFSET_FLUSH_INTERVAL_MS",
                    "value": "10000"
                  },
                  {
                    "name": "CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR",
                    "value": "1"
                  },
                  {
                    "name": "CONNECT_OFFSET_STORAGE_TOPIC",
                    "value": "docker-connect-offsets"
                  },
                  {
                    "name": "CONNECT_PLUGIN_PATH",
                    "value": "/usr/share/java,/usr/share/confluent-hub-components"
                  },
                  {
                    "name": "CONNECT_PRODUCER_INTERCEPTOR_CLASSES",
                    "value": "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
                  },
                  {
                    "name": "CONNECT_REST_ADVERTISED_HOST_NAME",
                    "value": "connect"
                  },
                  { "name": "CONNECT_REST_PORT", "value": "8083" },
                  {
                    "name": "CONNECT_STATUS_STORAGE_REPLICATION_FACTOR",
                    "value": "1"
                  },
                  {
                    "name": "CONNECT_STATUS_STORAGE_TOPIC",
                    "value": "docker-connect-status"
                  },
                  {
                    "name": "CONNECT_VALUE_CONVERTER",
                    "value": "io.confluent.connect.avro.AvroConverter"
                  },
                  {
                    "name": "CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL",
                    "value": "http://localhost:8081"
                  },
                  {
                    "name": "CONNECT_ZOOKEEPER_CONNECT",
                    "value": "localhost:2181"
                  }
                ]
              }
            ],
            "volumes": [{ "name": "kafka", "emptyDir": {} }],
            "restartPolicy": "Always",
            "terminationGracePeriodSeconds": 30,
            "dnsPolicy": "ClusterFirst",
            "securityContext": {},
            "schedulerName": "default-scheduler"
          }
        }
      }
    },
    {
      "kind": "Service",
      "apiVersion": "v1",
      "metadata": {
        "name": "lebron-confluent-kafka"
      },
      "spec": {
        "ports": [
          { "name": "9092", "port": 9092, "targetPort": 9092 },
          { "name": "2181", "port": 2181, "targetPort": 2181 },
          { "name": "8081", "port": 8081, "targetPort": 8081 },
          { "name": "8082", "port": 8082, "targetPort": 8082 },
          { "name": "8083", "port": 8083, "targetPort": 8083 },
          { "name": "8088", "port": 8088, "targetPort": 8088 }
        ],
        "selector": { "deploymentconfig": "lebron-confluent-kafka" },
        "type": "ClusterIP",
        "sessionAffinity": "None"
      },
      "status": { "loadBalancer": {} }
    }
  ]
}

Dockerfile

FROM confluentinc/cp-server-connect

RUN confluent-hub install debezium/debezium-connector-postgresql:0.10.0 --no-prompt

RUN confluent-hub install debezium/debezium-connector-mysql:1.1.0 --no-prompt