Docker-compose with Two Kafka Clusters - For Dev Machine - docker

Question

In production I will have two sites that each has a Kafka cluster and Postgres instance.
I would like to simulate that and create on my Dev machine two Kafka "clusters" each with a single broker and two Postgres instances. Each Kafka "cluster" is going to run a source connector which will copy data to the other site. The docker-compose below has two Postgres instances, but only one Kafka + source connector. What do I need to duplicate/change to create a second Kafka cluster with another source connector?
---
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper:6.0.0
hostname: zookeeper
container_name: zookeeper
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
networks:
- syncNetwork
broker:
image: confluentinc/cp-kafka:6.0.0
hostname: broker
container_name: broker
depends_on:
- zookeeper
ports:
- "29092:29092"
- "9092:9092"
- "9101:9101"
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JMX_PORT: 9101
networks:
- syncNetwork
schema-registry:
image: confluentinc/cp-schema-registry:6.0.0
hostname: schema-registry
container_name: schema-registry
depends_on:
- broker
ports:
- "8081:8081"
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
networks:
- syncNetwork
postgres12x_nms:
image: "postgres:12.4"
container_name: "pg12x_nms"
ports:
- "5430:5432"
environment:
POSTGRES_USER: "postgres"
POSTGRES_PASSWORD: "postgres"
POSTGRES_DB: "n_db"
volumes:
- vpg_nms:/var/lib/postgresql/data_nms
- ./schema_nms.sql:/docker-entrypoint-initdb.d/schema_nms.sql
networks:
- syncNetwork
pgadmin4:
image: dpage/pgadmin4
links:
- postgres12x_nms
networks:
- syncNetwork
postgres12x_mgr:
image: "postgres:12.4"
container_name: "pg12x_mgr"
ports:
- "5431:5432"
environment:
POSTGRES_USER: "postgres"
POSTGRES_PASSWORD: "postgres"
POSTGRES_DB: "n_db"
volumes:
- vpg_mgr:/var/lib/postgresql/data_mgr
- ./schema_mgr.sql:/docker-entrypoint-initdb.d/schema_mgr.sql
networks:
- syncNetwork
connect:
image: confluentinc/cp-kafka-connect
hostname: connect
container_name: connect
depends_on:
- broker
- schema-registry
ports:
- "8083:8083"
environment:
CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
networks:
- syncNetwork
command:
- bash
- -c
- |
confluent-hub install --no-prompt cjmatta/kafka-connect-irc:5.0.0
confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:latest
/etc/confluent/docker/run &
echo "Waiting for Kafka Connect to start listening on $$CONNECT_REST_ADVERTISED_HOST_NAME"
while [ $$(curl -s -o /dev/null -w %{http_code} http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors) -ne 200 ] ; do
echo -e $$(date) " Kafka Connect listener HTTP state: " $$(curl -s -o /dev/null -w %{http_code} http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors) " (waiting for 200)"
sleep 5
echo "Still waiting for Kafka Connect to start listening on $$CONNECT_REST_ADVERTISED_HOST_NAME"
done
echo "Completed - Still waiting for Kafka Connect to start listening on $$CONNECT_REST_ADVERTISED_HOST_NAME"
nc -vz $$CONNECT_REST_ADVERTISED_HOST_NAME $$CONNECT_REST_PORT
echo "Waiting for Schema Registry to start listening on schema-registry:8081"
while [ $$(curl -s -o /dev/null -w %{http_code} http://schema-registry:8081) -eq 000 ] ; do
echo -e $$(date) " Schema Registry listener HTTP state: " $$(curl -s -o /dev/null -w %{http_code} http://schema-registry:8081) " (waiting for != 000)"
sleep 5
echo "Still waiting for Schema Registry to start listening on schema-registry:8081"
done
echo "Completed - Still waiting for Schema Registry to start listening on schema-registry:8081"
#
echo -e "\n--\n+> Creating Kafka Connect JDBC Sink"
curl -X POST http://localhost:8083/connectors -H "Content-Type: application/json" -d '{
"name":"source_connector_cdc",
"config": {
"connector.class":"io.confluent.connect.jdbc.JdbcSourceConnector",
"connection.attempts": "1000000",
"connection.backoff.ms": "5000",
"tasks.max":"1",
"connection.url":"jdbc:postgresql://pg12x_nms:5432/n_db?user=postgres&password=postgres",
"mode":"timestamp",
"timestamp.column.name":"updated",
"table.types":"VIEW",
"table.whitelist":"hub_elements_v",
"topic.prefix":"cdc_",
"poll.interval.ms":"10000",
"timestamp.delay.interval.ms":"1000",
"validate.non.null":"false",
"quote.sql.identifiers":"never",
"key.converter":"org.apache.kafka.connect.json.JsonConverter",
"value.converter":"org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable":"false",
"value.converter.schemas.enable":"false",
"plugin.path":"/opt/confluent/share/java,/opt/confluent/share/java/kafka-connect-jdbc"
}
}'
#
sleep infinity
ksqldb-server:
image: confluentinc/cp-ksqldb-server:6.0.0
hostname: ksqldb-server
container_name: ksqldb-server
depends_on:
- broker
- connect
ports:
- "8088:8088"
environment:
KSQL_CONFIG_DIR: "/etc/ksql"
KSQL_BOOTSTRAP_SERVERS: "broker:29092"
KSQL_HOST_NAME: ksqldb-server
KSQL_LISTENERS: "http://0.0.0.0:8088"
KSQL_CACHE_MAX_BYTES_BUFFERING: 0
KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
KSQL_KSQL_CONNECT_URL: "http://connect:8083"
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: 'true'
KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: 'true'
networks:
- syncNetwork
ksqldb-cli:
image: confluentinc/cp-ksqldb-cli:6.0.0
container_name: ksqldb-cli
depends_on:
- broker
- connect
- connect
- ksqldb-server
entrypoint: /bin/sh
tty: true
networks:
- syncNetwork
rest-proxy:
image: confluentinc/cp-kafka-rest:6.0.0
depends_on:
- broker
- schema-registry
ports:
- 8082:8082
hostname: rest-proxy
container_name: rest-proxy
environment:
KAFKA_REST_HOST_NAME: rest-proxy
KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'
KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
networks:
- syncNetwork
networks:
syncNetwork:
external: true
volumes:
vpg_nms:
vpg_mgr:

Answer 1

Not clear what issues you are having copying the kafka and connect services.
But do so, and for Kafka2, you can use the same ZK container, but it needs a new chroot
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181/kafka2'
Then in the second connect image, you can modify CONNECT_BOOTSTRAP_SERVERS and its advertised name