We have been facing an issue while trying to spin up a local containerized confluent kafka ecosystem using docker-compose tool.
Host OS (Virtual Machine connecting through RDP)
$ sudo lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 18.04.6 LTS
Release: 18.04
Codename: bionic
$ uname -a
Linux IS*******1 4.15.0-189-generic #200-Ubuntu SMP Wed Jun 22 19:53:37 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
ghosh.sayak#ISRXDLKFD001:~$
Docker Engine
$ sudo docker version
Client: Docker Engine - Community
Version: 20.10.20
API version: 1.41
Go version: go1.18.7
Git commit: 9fdeb9c
Built: Tue Oct 18 18:20:19 2022
OS/Arch: linux/amd64
Context: default
Experimental: true
Server: Docker Engine - Community
Engine:
Version: 20.10.20
API version: 1.41 (minimum version 1.12)
Go version: go1.18.7
Git commit: 03df974
Built: Tue Oct 18 18:18:11 2022
OS/Arch: linux/amd64
Experimental: false
containerd:
Version: 1.6.16
GitCommit: 31aa4358a36870b21a992d3ad2bef29e1d693bec
runc:
Version: 1.1.4
GitCommit: v1.1.4-0-g5fd4c4d
docker-init:
Version: 0.19.0
GitCommit: de40ad0
docker-compose.yaml (Compose created network "kafka_default" using default bridge driver)
---
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper:7.3.0
hostname: zookeeper
container_name: zookeeper
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
broker:
image: confluentinc/cp-server:7.3.0
hostname: broker
container_name: broker
depends_on:
- zookeeper
ports:
- "9092:9092"
- "9101:9101"
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_JMX_PORT: 9101
KAFKA_JMX_HOSTNAME: localhost
KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
CONFLUENT_METRICS_ENABLE: 'true'
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
schema-registry:
image: confluentinc/cp-schema-registry:7.3.0
hostname: schema-registry
container_name: schema-registry
depends_on:
- broker
ports:
- "8081:8081"
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
connect:
image: cnfldemos/cp-server-connect-datagen:0.6.0-7.3.0
hostname: connect
container_name: connect
depends_on:
- broker
- schema-registry
ports:
- "8083:8083"
environment:
CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-6.2.0.jar
CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
CONNECT_PLUGIN_PATH: "/usr/share/java/ConnectPlugin"
CONNECT_LOG4J_ROOT_LOGLEVEL: INFO
CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
CONNECT_TOPIC_CREATION_ENABLE: 'false'
volumes:
- ./connect-plugins:/usr/share/java/ConnectPlugin
control-center:
image: confluentinc/cp-enterprise-control-center:7.3.0
hostname: control-center
container_name: control-center
depends_on:
- broker
- schema-registry
- connect
ports:
- "9021:9021"
environment:
CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092'
CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083'
CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
CONTROL_CENTER_REPLICATION_FACTOR: 1
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
CONFLUENT_METRICS_TOPIC_REPLICATION: 1
PORT: 9021
Now, while executing the command $ sudo docker-compose up -d, we could only observed that the "zookeeper" container is only getting Up and the "broker" i.e. Kafka container is Exited with Status 1 with the following error -
$ sudo docker container logs -t --details broker
2023-02-08T14:07:30.660123686Z ===> User
2023-02-08T14:07:30.662518281Z uid=1000(appuser) gid=1000(appuser) groups=1000(appuser)
2023-02-08T14:07:30.666677333Z ===> Configuring ...
2023-02-08T14:07:34.144645597Z ===> Running preflight checks ...
2023-02-08T14:07:34.147953062Z ===> Check if /var/lib/kafka/data is writable ...
2023-02-08T14:07:34.564183498Z ===> Check if Zookeeper is healthy ...
2023-02-08T14:07:35.861623527Z [2023-02-08 14:07:35,856] INFO Client environment:zookeeper.version=3.6.3--6401e4ad2087061bc6b9f80dec2d69f2e3c8660a, built on 04/08/2021 16:35 GMT (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861652177Z [2023-02-08 14:07:35,857] INFO Client environment:host.name=broker (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861683838Z [2023-02-08 14:07:35,857] INFO Client environment:java.version=11.0.16.1 (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861689083Z [2023-02-08 14:07:35,857] INFO Client environment:java.vendor=Azul Systems, Inc. (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861694000Z [2023-02-08 14:07:35,857] INFO Client environment:java.home=/usr/lib/jvm/zulu11-ca (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861703288Z [2023-02-08 14:07:35,857] INFO Client environment:java.class.path=/usr/share/java/cp-base-new/slf4j-api-1.7.36.jar:/usr/share/java/cp-base-new/disk-usage-agent-7.3.0.jar:/usr/share/java/cp-base-new/paranamer-2.8.jar:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.14.0.jar:/usr/share/java/cp-base-new/jackson-annotations-2.13.2.jar:/usr/share/java/cp-base-new/metrics-core-2.2.0.jar:/usr/share/java/cp-base-new/jolokia-core-1.7.1.jar:/usr/share/java/cp-base-new/slf4j-reload4j-1.7.36.jar:/usr/share/java/cp-base-new/re2j-1.6.jar:/usr/share/java/cp-base-new/snakeyaml-1.30.jar:/usr/share/java/cp-base-new/utility-belt-7.3.0.jar:/usr/share/java/cp-base-new/zookeeper-jute-3.6.3.jar:/usr/share/java/cp-base-new/audience-annotations-0.5.0.jar:/usr/share/java/cp-base-new/scala-collection-compat_2.13-2.6.0.jar:/usr/share/java/cp-base-new/kafka-metadata-7.3.0-ccs.jar:/usr/share/java/cp-base-new/argparse4j-0.7.0.jar:/usr/share/java/cp-base-new/kafka-storage-api-7.3.0-ccs.jar:/usr/share/java/cp-base-new/jopt-simple-5.0.4.jar:/usr/share/java/cp-base-new/common-utils-7.3.0.jar:/usr/share/java/cp-base-new/jackson-dataformat-yaml-2.13.2.jar:/usr/share/java/cp-base-new/logredactor-1.0.10.jar:/usr/share/java/cp-base-new/zookeeper-3.6.3.jar:/usr/share/java/cp-base-new/zstd-jni-1.5.2-1.jar:/usr/share/java/cp-base-new/kafka-raft-7.3.0-ccs.jar:/usr/share/java/cp-base-new/kafka-server-common-7.3.0-ccs.jar:/usr/share/java/cp-base-new/scala-logging_2.13-3.9.4.jar:/usr/share/java/cp-base-new/jose4j-0.7.9.jar:/usr/share/java/cp-base-new/snappy-java-1.1.8.4.jar:/usr/share/java/cp-base-new/scala-reflect-2.13.5.jar:/usr/share/java/cp-base-new/scala-library-2.13.5.jar:/usr/share/java/cp-base-new/scala-java8-compat_2.13-1.0.2.jar:/usr/share/java/cp-base-new/jackson-core-2.13.2.jar:/usr/share/java/cp-base-new/minimal-json-0.9.5.jar:/usr/share/java/cp-base-new/kafka-clients-7.3.0-ccs.jar:/usr/share/java/cp-base-new/jackson-dataformat-csv-2.13.2.jar:/usr/share/java/cp-base-new/jackson-datatype-jdk8-2.13.2.jar:/usr/share/java/cp-base-new/kafka-storage-7.3.0-ccs.jar:/usr/share/java/cp-base-new/lz4-java-1.8.0.jar:/usr/share/java/cp-base-new/jolokia-jvm-1.7.1.jar:/usr/share/java/cp-base-new/jackson-module-scala_2.13-2.13.2.jar:/usr/share/java/cp-base-new/kafka_2.13-7.3.0-ccs.jar:/usr/share/java/cp-base-new/jackson-databind-2.13.2.2.jar:/usr/share/java/cp-base-new/logredactor-metrics-1.0.10.jar:/usr/share/java/cp-base-new/gson-2.9.0.jar:/usr/share/java/cp-base-new/json-simple-1.1.1.jar:/usr/share/java/cp-base-new/reload4j-1.2.19.jar:/usr/share/java/cp-base-new/metrics-core-4.1.12.1.jar:/usr/share/java/cp-base-new/commons-cli-1.4.jar (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861725300Z [2023-02-08 14:07:35,857] INFO Client environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861730555Z [2023-02-08 14:07:35,857] INFO Client environment:java.io.tmpdir=/tmp (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861734946Z [2023-02-08 14:07:35,857] INFO Client environment:java.compiler=<NA> (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861740934Z [2023-02-08 14:07:35,857] INFO Client environment:os.name=Linux (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861745386Z [2023-02-08 14:07:35,857] INFO Client environment:os.arch=amd64 (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861749485Z [2023-02-08 14:07:35,857] INFO Client environment:os.version=4.15.0-189-generic (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.861754326Z [2023-02-08 14:07:35,857] INFO Client environment:user.name=appuser (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.862778083Z [2023-02-08 14:07:35,857] INFO Client environment:user.home=/home/appuser (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.862781895Z [2023-02-08 14:07:35,857] INFO Client environment:user.dir=/home/appuser (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.862785144Z [2023-02-08 14:07:35,857] INFO Client environment:os.memory.free=242MB (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.862788600Z [2023-02-08 14:07:35,857] INFO Client environment:os.memory.max=4006MB (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.862792192Z [2023-02-08 14:07:35,857] INFO Client environment:os.memory.total=252MB (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.875374527Z [2023-02-08 14:07:35,869] INFO Initiating client connection, connectString=zookeeper:2181 sessionTimeout=40000 watcher=io.confluent.admin.utils.ZookeeperConnectionWatcher#3c0a50da (org.apache.zookeeper.ZooKeeper)
2023-02-08T14:07:35.875395839Z [2023-02-08 14:07:35,873] INFO Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation (org.apache.zookeeper.common.X509Util)
2023-02-08T14:07:35.898794157Z [2023-02-08 14:07:35,894] INFO jute.maxbuffer value is 1048575 Bytes (org.apache.zookeeper.ClientCnxnSocket)
2023-02-08T14:07:35.910713716Z [2023-02-08 14:07:35,905] INFO zookeeper.request.timeout value is 0. feature enabled=false (org.apache.zookeeper.ClientCnxn)
2023-02-08T14:07:36.094475506Z [2023-02-08 14:07:36,087] INFO Opening socket connection to server zookeeper/172.19.0.2:2181. (org.apache.zookeeper.ClientCnxn)
2023-02-08T14:07:36.094498351Z [2023-02-08 14:07:36,090] INFO SASL config status: Will not attempt to authenticate using SASL (unknown error) (org.apache.zookeeper.ClientCnxn)
2023-02-08T14:07:36.114680668Z [2023-02-08 14:07:36,111] ERROR Unable to open socket to zookeeper/172.19.0.2:2181 (org.apache.zookeeper.ClientCnxnSocketNIO)
2023-02-08T14:07:36.121566533Z [2023-02-08 14:07:36,112] WARN Session 0x0 for sever zookeeper/172.19.0.2:2181, Closing socket connection. Attempting reconnect except it is a SessionExpiredException. (org.apache.zookeeper.ClientCnxn)
2023-02-08T14:07:36.121606559Z java.net.ConnectException: Connection refused
2023-02-08T14:07:36.121610572Z at java.base/sun.nio.ch.Net.connect0(Native Method)
2023-02-08T14:07:36.121614024Z at java.base/sun.nio.ch.Net.connect(Net.java:483)
2023-02-08T14:07:36.121617448Z at java.base/sun.nio.ch.Net.connect(Net.java:472)
2023-02-08T14:07:36.121620610Z at java.base/sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:692)
2023-02-08T14:07:36.121623850Z at org.apache.zookeeper.ClientCnxnSocketNIO.registerAndConnect(ClientCnxnSocketNIO.java:260)
2023-02-08T14:07:36.121627034Z at org.apache.zookeeper.ClientCnxnSocketNIO.connect(ClientCnxnSocketNIO.java:270)
2023-02-08T14:07:36.121630306Z at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1177)
2023-02-08T14:07:36.121633562Z at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1210)
2023-02-08T14:07:37.230453807Z [2023-02-08 14:07:37,224] INFO Opening socket connection to server zookeeper/172.19.0.2:2181. (org.apache.zookeeper.ClientCnxn)
2023-02-08T14:07:37.230488726Z [2023-02-08 14:07:37,224] INFO SASL config status: Will not attempt to authenticate using SASL (unknown error) (org.apache.zookeeper.ClientCnxn)
2023-02-08T14:07:37.230492954Z [2023-02-08 14:07:37,224] ERROR Unable to open socket to zookeeper/172.19.0.2:2181 (org.apache.zookeeper.ClientCnxnSocketNIO)
2023-02-08T14:07:37.230496368Z [2023-02-08 14:07:37,225] WARN Session 0x0 for sever zookeeper/172.19.0.2:2181, Closing socket connection. Attempting reconnect except it is a SessionExpiredException. (org.apache.zookeeper.ClientCnxn)
To debug the connectivity, I have done couple of flight checks like -
Created a netshoot container within same "kafka_default" network using sudo docker run -it --net container:zookeeper nicolaka/netshoot
Some useful information -
zookeeper# cat /etc/hosts
127.0.0.1 localhost
::1 localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
172.19.0.2 zookeeper
zookeeper# cat /etc/resolv.conf
search corp.***.com
nameserver 127.0.0.11
options ndots:0
Tried nslookup, netcat, ping, nmap, telnet from within the container and the results are following -
zookeeper# nslookup zookeeper
Server: 127.0.0.11
Address: 127.0.0.11#53
Non-authoritative answer:
Name: zookeeper
Address: 172.19.0.2
zookeeper# nc -v -l -p 2181
nc: Address in use
zookeeper# ping -c 2 zookeeper
PING zookeeper (172.19.0.2) 56(84) bytes of data.
64 bytes from zookeeper (172.19.0.2): icmp_seq=1 ttl=64 time=0.054 ms
64 bytes from zookeeper (172.19.0.2): icmp_seq=2 ttl=64 time=0.049 ms
--- zookeeper ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1015ms
rtt min/avg/max/mdev = 0.049/0.051/0.054/0.002 ms
zookeeper# nmap -p0- -v -A -T4 zookeeper
Starting Nmap 7.93 ( https://nmap.org ) at 2023-02-08 15:43 UTC
NSE: Loaded 155 scripts for scanning.
NSE: Script Pre-scanning.
Initiating NSE at 15:43
Completed NSE at 15:43, 0.00s elapsed
Initiating NSE at 15:43
Completed NSE at 15:43, 0.00s elapsed
Initiating NSE at 15:43
Completed NSE at 15:43, 0.00s elapsed
Initiating SYN Stealth Scan at 15:43
Scanning zookeeper (172.19.0.2) [65536 ports]
Completed SYN Stealth Scan at 15:43, 2.96s elapsed (65536 total ports)
Initiating Service scan at 15:43
Initiating OS detection (try #1) against zookeeper (172.19.0.2)
Retrying OS detection (try #2) against zookeeper (172.19.0.2)
NSE: Script scanning 172.19.0.2.
Initiating NSE at 15:43
Completed NSE at 15:43, 0.01s elapsed
Initiating NSE at 15:43
Completed NSE at 15:43, 0.00s elapsed
Initiating NSE at 15:43
Completed NSE at 15:43, 0.00s elapsed
Nmap scan report for zookeeper (172.19.0.2)
Host is up (0.000060s latency).
Not shown: 65533 closed tcp ports (reset)
PORT STATE SERVICE VERSION
2181/tcp filtered eforward
8080/tcp filtered http-proxy
39671/tcp filtered unknown
Too many fingerprints match this host to give specific OS details
Network Distance: 0 hops
NSE: Script Post-scanning.
Initiating NSE at 15:43
Completed NSE at 15:43, 0.00s elapsed
Initiating NSE at 15:43
Completed NSE at 15:43, 0.00s elapsed
Initiating NSE at 15:43
Completed NSE at 15:43, 0.00s elapsed
Read data files from: /usr/bin/../share/nmap
OS and Service detection performed. Please report any incorrect results at https://nmap.org/submit/ .
Nmap done: 1 IP address (1 host up) scanned in 5.55 seconds
Raw packets sent: 65551 (2.885MB) | Rcvd: 131094 (5.508MB)
zookeeper# telnet localhost 2181
Connected to localhost
***zookeeper# telnet zookeeper 2181
telnet: can't connect to remote host (172.19.0.2): Connection refused***
BUT, WHEN, tried from Host Machine, it succeeded -
ghosh.sayak#IS********1:~/Kafka$ telnet localhost 2181
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
^]
telnet> q
Connection closed.
Also, here is the current status of the container -
ghosh.sayak#IS********1:~/Kafka$ sudo docker-compose ps
[sudo] password for ghosh.sayak:
Name Command State Ports
-------------------------------------------------------------------------------------------------------------------
broker /etc/confluent/docker/run Exit 1
connect /etc/confluent/docker/run Exit 1
control-center /etc/confluent/docker/run Exit 1
schema-registry /etc/confluent/docker/run Exit 1
zookeeper /etc/confluent/docker/run Up 0.0.0.0:2181->2181/tcp,:::2181->2181/tcp, 2888/tcp, 3888/tcp
NOTE: Docker Engine has been freshly installed by following this official documentation.
Stuck with this issue! Any help will be much appreciated!
Thanks in advance!
Related
I need help! (who would have thought, right? lol)
I have a job interview in few days and it would mean the world to me to be well prepared for it and have some working examples.
I am trying to set up an ELK pipeline to stream data from kafka, through logstash, elasticsearch and finally read it from Kibana. The usual.
I am making use of containers, but the duo logstash - elasticsearch are giving me an aneurism.
Everything else works perfectly fine. I've checked the logs off of kafka and that is working just fine. Kibana is collected to elasticsearch just fine as well. But logstash and es really don't want to match.
Here is the setup
docker-compose.yml
version: '3.6'
services:
elasticsearch:
image: elasticsearch:8.6.0
container_name: elasticsearch
#restart: always
volumes:
- elastic_data:/usr/share/elasticsearch/data/
environment:
cluster.name: elf-kafka-cluster
ES_JAVA_OPTS: "-Xmx256m -Xms256m"
discovery.type: single-node
xpack.security.enabled: false
ports:
- '9200:9200'
- '9300:9300'
networks:
- elk
kibana:
image: kibana:8.6.0
container_name: kibana
#restart: always
ports:
- '5601:5601'
environment:
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
depends_on:
- elasticsearch
networks:
- elk
logstash:
image: logstash:8.6.0
container_name: logstash
#restart: always
volumes:
- type: bind
source: ./logstash_pipeline/
target: /usr/share/logstash/pipeline
read_only: true
command: logstash -f /home/ettore/Documenti/Portfolio/ELK/logstash/logstash.conf
depends_on:
- elasticsearch
ports:
- '9600:9600'
environment:
xpack.monitoring.enabled: true
# LS_JAVA_OPTS: "-Xmx256m -Xms256m"
links:
- elasticsearch
networks:
- elk
volumes:
elastic_data: {}
networks:
elk:
driver: bridge
logstash.conf
input {
kafka {
bootstrap_servers => "localhost:9092"
topics => ["topic"]
}
}
output {
elasitcsearch {
hosts => ["http://localhost:9200"]
index => "topic"
workers => 1
}
}
These are logstash error logs when I compose up:
logstash | [2023-01-17T13:59:02,680][WARN ][deprecation.logstash.monitoringextension.pipelineregisterhook] Internal collectors option for Logstash monitoring is deprecated and targeted for removal in the next major version.
logstash | Please configure Metricbeat to monitor Logstash. Documentation can be found at:
logstash | https://www.elastic.co/guide/en/logstash/current/monitoring-with-metricbeat.html
logstash | [2023-01-17T13:59:04,711][INFO ][logstash.licensechecker.licensereader] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elasticsearch:9200/]}}
logstash | [2023-01-17T13:59:05,373][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"Connect to elasticsearch:9200 [elasticsearch/172.20.0.2] failed: Connection refused", :exception=>Manticore::SocketException, :cause=>#<Java::OrgApacheHttpConn::HttpHostConnectException: Connect to elasticsearch:9200 [elasticsearch/172.20.0.2] failed: Connection refused>}
logstash | [2023-01-17T13:59:05,379][WARN ][logstash.licensechecker.licensereader] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"http://elasticsearch:9200/", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :message=>"Elasticsearch Unreachable: [http://elasticsearch:9200/][Manticore::SocketException] Connect to elasticsearch:9200 [elasticsearch/172.20.0.2] failed: Connection refused"}
logstash | [2023-01-17T13:59:05,436][INFO ][logstash.licensechecker.licensereader] Failed to perform request {:message=>"Connect to elasticsearch:9200 [elasticsearch/172.20.0.2] failed: Connection refused", :exception=>Manticore::SocketException, :cause=>#<Java::OrgApacheHttpConn::HttpHostConnectException: Connect to elasticsearch:9200 [elasticsearch/172.20.0.2] failed: Connection refused>}
logstash | [2023-01-17T13:59:05,444][WARN ][logstash.licensechecker.licensereader] Marking url as dead. Last error: [LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [http://elasticsearch:9200/_xpack][Manticore::SocketException] Connect to elasticsearch:9200 [elasticsearch/172.20.0.2] failed: Connection refused {:url=>http://elasticsearch:9200/, :error_message=>"Elasticsearch Unreachable: [http://elasticsearch:9200/_xpack][Manticore::SocketException] Connect to elasticsearch:9200 [elasticsearch/172.20.0.2] failed: Connection refused", :error_class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError"}
logstash | [2023-01-17T13:59:05,449][WARN ][logstash.licensechecker.licensereader] Attempt to validate Elasticsearch license failed. Sleeping for 0.02 {:fail_count=>1, :exception=>"Elasticsearch Unreachable: [http://elasticsearch:9200/_xpack][Manticore::SocketException] Connect to elasticsearch:9200 [elasticsearch/172.20.0.2] failed: Connection refused"}
logstash | [2023-01-17T13:59:05,477][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"}
logstash | [2023-01-17T13:59:05,567][ERROR][logstash.monitoring.internalpipelinesource] Failed to fetch X-Pack information from Elasticsearch. This is likely due to failure to reach a live Elasticsearch cluster.
logstash | [2023-01-17T13:59:05,661][INFO ][logstash.config.source.local.configpathloader] No config files found in path {:path=>"/home/ettore/Documenti/Portfolio/ELK/logstash/logstash.conf"}
logstash | [2023-01-17T13:59:05,664][ERROR][logstash.config.sourceloader] No configuration found in the configured sources.
logstash | [2023-01-17T13:59:06,333][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600, :ssl_enabled=>false}
logstash | [2023-01-17T13:59:06,411][INFO ][logstash.runner ] Logstash shut down.
logstash | [2023-01-17T13:59:06,419][FATAL][org.logstash.Logstash ] Logstash stopped processing because of an error: (SystemExit) exit
logstash | org.jruby.exceptions.SystemExit: (SystemExit) exit
logstash | at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:790) ~[jruby.jar:?]
logstash | at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:753) ~[jruby.jar:?]
logstash | at usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb:91) ~[?:?]
and this is to prove that everything is working as intended with es (or so it seems)
netstat -an | grep 9200
tcp 0 0 0.0.0.0:9200 0.0.0.0:* LISTEN
tcp6 0 0 :::9200 :::* LISTEN
unix 3 [ ] STREAM CONNECTED 49200
I've looked through everything and this is 100% not a duplicate because I have tried it all. I really can't figure it out. Hope anyone can help.
Thank you for you time.
You should set logstash.yml
Create a logstash.yml with values below:
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://localhost:9200" ]
In your docker-compose.yml, add another volume in Logstash container as shown below:
./logstash.yml:/usr/share/logstash/config/logstash.yml
Additionally, its good to run with restart condition.
I have the following docker-compose file which is a copy of the docker-compose from the docker apache flink site. The only difference is that I am using the Mac m1 version.
version: "2.2"
services:
jobmanager:
image: arm64v8/flink:alpine
ports:
- "8081:8081"
command: standalone-job --job-classname com.job.ClassName [--job-id <job id>] [--fromSavepoint /path/to/savepoint [--allowNonRestoredState]] [job arguments]
volumes:
- ~/sg_flink/artifacts:/opt/flink/usrlib
networks:
- flink-network
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: jobmanager
parallelism.default: 2
taskmanager:
image: arm64v8/flink:alpine
depends_on:
- jobmanager
command: taskmanager
scale: 1
volumes:
- ~/sg_flink/artifacts:/opt/flink/usrlib
networks:
- flink-network
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: jobmanager
taskmanager.numberOfTaskSlots: 2
parallelism.default: 2
networks:
flink-network:
The error is a connection is refused
taskmanager_1 | 2021-11-03 17:43:02,724 INFO org.apache.flink.runtime.taskexecutor.TaskExecutor - Could not resolve ResourceManager address akka.tcp://flink#9cf35ea13c8b:6123/user/resourcemanager, retrying in 10000 ms: Could not connect to rpc endpoint under address akka.tcp://flink#9cf35ea13c8b:6123/user/resourcemanager..
taskmanager_1 | 2021-11-03 17:43:12,753 WARN akka.remote.transport.netty.NettyTransport - Remote connection to [null] failed with java.net.ConnectException: Connection refused: 9cf35ea13c8b/172.20.0.3:6123
taskmanager_1 | 2021-11-03 17:43:12,756 WARN akka.remote.ReliableDeliverySupervisor - Association with remote system [akka.tcp://flink#9cf35ea13c8b:6123] has failed, address is now gated for [50] ms. Reason: [Association failed with [akka.tcp://flink#9cf35ea13c8b:6123]] Caused by: [Connection refused: 9cf35ea13c8b/172.20.0.3:6123]
taskmanager_1 | 2021-11-03 17:43:12,758 INFO org.apache.flink.runtime.taskexecutor.TaskExecutor - Could not resolve ResourceManager address akka.tcp://flink#9cf35ea13c8b:6123/user/resourcemanager, retrying in 10000 ms: Could not connect to rpc endpoint under address akka.tcp://flink#9cf35ea13c8b:6123/user/resourcemanager..
docker ps output looks like this
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
4416f88f60c2 arm64v8/flink:alpine "/docker-entrypoint.…" 44 seconds ago Up 44 seconds 6123/tcp, 8081/tcp sg_flink_taskmanager_1
c211940acf41 arm64v8/flink:alpine "/docker-entrypoint.…" 45 seconds ago Up 44 seconds 6123/tcp, 0.0.0.0:8081->8081/tcp sg_flink_jobmanager_1
```
I have met this trouble in same condittion,and I fixed it use docker-compse links.
taskmanager:
links:
- jobmanager
I'm working on Mac with Docker Desktop. I'm trying to run wurstmeister/kafka from docker compose and connect a producer to it.
This is my docker-compose.yml:
version: '3.8'
services:
zookeeper:
container_name: zookeeper
image: zookeeper:3.7.0
ports:
- "2181:2181"
environment:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=zookeeper:2888:3888;2181
restart: on-failure
kafka:
container_name: kafka
image: wurstmeister/kafka:2.13-2.7.0
ports:
- "9092:9092"
environment:
KAFKA_LISTENERS: INTERNAL://kafka:19092,EXTERNAL://localhost:9092
KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:19092,EXTERNAL://localhost:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_BROKER_ID: 1
restart: on-failure
depends_on:
- zookeeper
Then I have producer connecting to localhost:9092 and sending a simple message. The producer works fine - tested with another kafka image confluentinc/cp-kafka:6.2.0.
When I try to use producer with wurstmeister/kafka I'm getting a lot of this errors:
22:07:13.421 [kafka-producer-network-thread | simple-producer] DEBUG org.apache.kafka.clients.NetworkClient - [Producer clientId=simple-producer] Initialize connection to node localhost:9092 (id: -1 rack: null) for sending metadata request
22:07:13.421 [kafka-producer-network-thread | simple-producer] DEBUG org.apache.kafka.clients.NetworkClient - [Producer clientId=simple-producer] Initiating connection to node localhost:9092 (id: -1 rack: null) using address localhost/127.0.0.1
22:07:13.421 [kafka-producer-network-thread | simple-producer] DEBUG org.apache.kafka.common.network.Selector - [Producer clientId=simple-producer] Created socket with SO_RCVBUF = 326640, SO_SNDBUF = 146988, SO_TIMEOUT = 0 to node -1
22:07:13.421 [kafka-producer-network-thread | simple-producer] DEBUG org.apache.kafka.clients.NetworkClient - [Producer clientId=simple-producer] Completed connection to node -1. Fetching API versions.
22:07:13.421 [kafka-producer-network-thread | simple-producer] DEBUG org.apache.kafka.clients.NetworkClient - [Producer clientId=simple-producer] Initiating API versions fetch from node -1.
22:07:13.422 [kafka-producer-network-thread | simple-producer] DEBUG org.apache.kafka.clients.NetworkClient - [Producer clientId=simple-producer] Sending API_VERSIONS request with header RequestHeader(apiKey=API_VERSIONS, apiVersion=3, clientId=simple-producer, correlationId=20) and timeout 30000 to node -1: {client_software_name=apache-kafka-java,client_software_version=2.7.0,_tagged_fields={}}
22:07:13.423 [kafka-producer-network-thread | simple-producer] DEBUG org.apache.kafka.common.network.Selector - [Producer clientId=simple-producer] Connection with localhost/127.0.0.1 disconnected
java.io.EOFException: null
at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:97)
at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:447)
at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:397)
at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:674)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:576)
at org.apache.kafka.common.network.Selector.poll(Selector.java:481)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:561)
at org.apache.kafka.clients.producer.internals.Sender.runOnce(Sender.java:325)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:240)
at java.base/java.lang.Thread.run(Thread.java:829)
22:07:13.424 [kafka-producer-network-thread | simple-producer] DEBUG org.apache.kafka.clients.NetworkClient - [Producer clientId=simple-producer] Node -1 disconnected.
22:07:13.424 [kafka-producer-network-thread | simple-producer] WARN org.apache.kafka.clients.NetworkClient - [Producer clientId=simple-producer] Bootstrap broker localhost:9092 (id: -1 rack: null) disconnected
Why this happens? What is the cause of this error? And how can make it work?
EDIT: added kafka container logs
Kafka container last logs below and no new logs added when I try to connect producer:
[2021-09-09 21:07:28,227] INFO Kafka version: 2.7.0 (org.apache.kafka.common.utils.AppInfoParser)
[2021-09-09 21:07:28,230] INFO Kafka commitId: 448719dc99a19793 (org.apache.kafka.common.utils.AppInfoParser)
[2021-09-09 21:07:28,231] INFO Kafka startTimeMs: 1631221648211 (org.apache.kafka.common.utils.AppInfoParser)
[2021-09-09 21:07:28,238] INFO [KafkaServer id=1] started (kafka.server.KafkaServer)
[2021-09-09 21:07:28,365] INFO [broker-1-to-controller-send-thread]: Recorded new controller, from now on will use broker 1 (kafka.server.BrokerToControllerRequestThread)
I would like to run two instances of rabbitmq on one server. All I create with docker-compose. The thing is how I can change the default node and management ports. I have tried setting it via ports but it didn't help. When I was facing the same scenario but with mongo, I have used command: mongod --port CUSTOM_PORT . What would be the analogical command here for rabbitmq?
Here is my config for the second instance of rabbitmq.
version: '2'
services:
rabbitmq:
image: rabbitmq:3-management-alpine
container_name: 'rabbitmq_test'
ports:
- 5673:5673
- 15673:15673
volumes:
- ./rabbitmq/data/:/var/lib/rabbitmq/
- ./rabbitmq/log/:/var/log/rabbitmq
networks:
- rabbitmq_go_net_test
environment:
RABBITMQ_DEFAULT_USER: 'test'
RABBITMQ_DEFAULT_PASS: 'test'
HOST_PORT_RABBIT: 5673
HOST_PORT_RABBIT_MGMT: 15673
networks:
rabbitmq_go_net_test:
driver: bridge
And the outcome is below
Management plugin: HTTP (non-TLS) listener started on port 15672
rabbitmq_test | 2021-03-18 11:32:42.553 [info] <0.738.0> Ready to start client connection listeners
rabbitmq_test | 2021-03-18 11:32:42.553 [info] <0.44.0> Application rabbitmq_prometheus started on node rabbit#fb24038613f3
rabbitmq_test | 2021-03-18 11:32:42.557 [info] <0.1035.0> started TCP listener on [::]:5672
We can see that there are still ports 5672 and 15672 exposed instead of 5673 and 15673.
EDIT
ports:
- 5673:5672
- 15673:15672
I have tried that the above conf yet with no success
rabbitmq_test | 2021-03-18 14:08:56.167 [info] <0.797.0> Management plugin: HTTP (non-TLS) listener started on port 15672
rabbitmq_test | 2021-03-18 14:08:56.167 [info] <0.903.0> Statistics database started.
rabbitmq_test | 2021-03-18 14:08:56.167 [info] <0.902.0> Starting worker pool 'management_worker_pool' with 3 processes in it
rabbitmq_test | 2021-03-18 14:08:56.168 [info] <0.44.0> Application rabbitmq_management started on node rabbit#9358e6f4d2a5
rabbitmq_test | 2021-03-18 14:08:56.208 [info] <0.44.0> Application prometheus started on node rabbit#9358e6f4d2a5
rabbitmq_test | 2021-03-18 14:08:56.213 [info] <0.916.0> Prometheus metrics: HTTP (non-TLS) listener started on port 15692
rabbitmq_test | 2021-03-18 14:08:56.213 [info] <0.44.0> Application rabbitmq_prometheus started on node rabbit#9358e6f4d2a5
rabbitmq_test | 2021-03-18 14:08:56.213 [info] <0.738.0> Ready to start client connection listeners
rabbitmq_test | 2021-03-18 14:08:56.216 [info] <0.1035.0> started TCP listener on [::]:5672
I have found the solution. I provided the configuration file to the rabbitmq container.
loopback_users.guest = false
listeners.tcp.default = 5673
default_pass = test
default_user = test
management.tcp.port = 15673
And a working docker-compose file
version: '2'
services:
rabbitmq:
image: rabbitmq:3-management-alpine
container_name: 'rabbitmq_test'
ports:
- 5673:5673
- 15673:15673
volumes:
- ./rabbitmq/data/:/var/lib/rabbitmq/
- ./rabbitmq/log/:/var/log/rabbitmq
- ./conf/myrabbit.conf:/etc/rabbitmq/rabbitmq.conf
networks:
- rabbitmq_go_net_test
networks:
rabbitmq_go_net_test:
driver: bridge
A working example with rabbitmq:3.9.13-management-alpine
docker/rabbitmq/rabbitmq.conf:
loopback_users.guest = false
listeners.tcp.default = 5673
default_pass = guest
default_user = guest
default_vhost = /
docker/rabbitmq/Dockerfile:
FROM rabbitmq:3.9.13-management-alpine
COPY --chown=rabbitmq:rabbitmq rabbitmq.conf /etc/rabbitmq/rabbitmq.conf
EXPOSE 4369 5671 5672 5673 15691 15692 25672 25673
docker-compose.yml:
...
rabbitmq:
#image: "rabbitmq:3-management-alpine"
build: './docker/rabbitmq/'
container_name: my-rabbitmq
environment:
RABBITMQ_DEFAULT_VHOST: /
ports:
- 5673:5672
- 15673:15672
networks:
- default
...
I'm testing a sample spring cloud stream application (running on a Ubuntu linux machine) with one source and one sink services. All my services are docker-containerized and I would like to use kafka as message broker.
Below the relevant parts of the docker-compose.yml:
zookeeper:
image: confluent/zookeeper
container_name: zookeeper
ports:
- "2181:2181"
kafka:
image: wurstmeister/kafka:0.9.0.0-1
container_name: kafka
ports:
- "9092:9092"
links:
- zookeeper:zk
environment:
- KAFKA_ADVERTISED_HOST_NAME=192.168.33.101
- KAFKA_ADVERTISED_PORT=9092
- KAFKA_DELETE_TOPIC_ENABLE=true
- KAFKA_LOG_RETENTION_HOURS=1
- KAFKA_MESSAGE_MAX_BYTES=10000000
- KAFKA_REPLICA_FETCH_MAX_BYTES=10000000
- KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS=60000
- KAFKA_NUM_PARTITIONS=2
- KAFKA_DELETE_RETENTION_MS=1000
.
.
.
# not shown: eureka service registry, spring cloud config service, etc.
myapp-service-test-source:
container_name: myapp-service-test-source
image: myapp-h2020/myapp-service-test-source:0.0.1
environment:
SERVICE_REGISTRY_HOST: 192.168.33.101
SERVICE_REGISTRY_PORT: 8761
ports:
- 8081:8080
.
.
.
Here the relevant part of application.yml for my service-test-source service:
spring:
cloud:
stream:
defaultBinder: kafka
bindings:
output:
destination: messages
content-type: application/json
kafka:
binder:
brokers: ${SERVICE_REGISTRY_HOST:192.168.33.101}
zkNodes: ${SERVICE_REGISTRY_HOST:192.168.33.101}
defaultZkPort: 2181
defaultBrokerPort: 9092
The problem is the following, if I launch the docker-compose above, in the test-source container log I notice that the service fails to connect to zookeeper, giving a repeated set of Connection refused error, and finishing with a ZkTimeoutException which makes the service terminate (see below).
The strange fact is that, if instead of running my source (and sink) test services as docker containers I run them as jar files via maven mvn spring-boot:run <etc...> the services work fine and are able to exchange messages via kafka. (note that kafka, zookeeper, etc. are still running as docker containers).
.
.
.
*** THE FOLLOWING REPEATED n TIMES ***
2017-02-14 14:40:09.164 INFO 1 --- [localhost:2181)] org.apache.zookeeper.ClientCnxn : Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error)
2017-02-14 14:40:09.166 WARN 1 --- [localhost:2181)] org.apache.zookeeper.ClientCnxn : Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[na:1.8.0_111]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) ~[na:1.8.0_111]
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361) ~[zookeeper-3.4.6.jar!/:3.4.6-1569965]
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) ~[zookeeper-3.4.6.jar!/:3.4.6-1569965]
.
.
.
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:53)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.springframework.context.ApplicationContextException: Failed to start bean 'outputBindingLifecycle'; nested exception is org.I0Itec.zkclient.exception.ZkTimeoutException: Unable to connect to zookeeper server within timeout: 10000
Any idea what the problem might be?
edit:
I discovered that in the "jar" execution logs the test-source service tries to connect to zookeeper through the IP 127.0.0.1, as can be seen from the log snipped below:
2017-02-15 14:24:04.159 INFO 10348 --- [localhost:2181)] org.apache.zookeeper.ClientCnxn : Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error)
2017-02-15 14:24:04.159 INFO 10348 --- [localhost:2181)] org.apache.zookeeper.ClientCnxn : Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error)
2017-02-15 14:24:04.178 INFO 10348 --- [localhost:2181)] org.apache.zookeeper.ClientCnxn : Socket connection established to localhost/127.0.0.1:2181, initiating session
2017-02-15 14:24:04.201 INFO 10348 --- [localhost:2181)] org.apache.zookeeper.ClientCnxn : Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x15a421fd9ec000a, negotiated timeout = 10000
2017-02-15 14:24:05.870 INFO 10348 --- [ main] org.apache.zookeeper.ZooKeeper : Initiating client connection, connectString=localhost:2181 sessionTimeout=6000 watcher=org.I0Itec.zkclient.ZkClient#72ba68e3
2017-02-15 14:24:05.882 INFO 10348 --- [localhost:2181)] org.apache.zookeeper.ClientCnxn : Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error)
2017-02-15 14:24:05.883 INFO 10348 --- [localhost:2181)] org.apache.zookeeper.ClientCnxn : Socket connection established to localhost/0:0:0:0:0:0:0:1:2181, initiating session
This explains why everything works on the jar execution but not the docker one (the zookeeper container exports its 2181 port to the host machine, so it's visible as localhost for the service process when running directly on the host machine), but doesn't solve the problem: Apparently the spring cloud stream kafka configuration is ignoring the property spring.cloud.stream.kafka.binder.zkNodes as set in the application.yml (note that if I log the value of such environment variable from the service, I see the correct value of 192.168.33.101 that I hardcoded there for debugging purposes).
You have set the defaultBinder to be rabbit while trying to use the Kafka binder configuration. Do you have both rabbit and kafka binders in the classpath of your application? In that case, you can enable here
zookeeper:
image: wurstmeister/zookeeper
container_name: 'zookeeper'
ports:
- 2181:2181
--------------------- kafka --------------------------------
kafka:
image: wurstmeister/kafka
container_name: 'kafka'
environment:
- KAFKA_ADVERTISED_HOST_NAME=kafka
- KAFKA_ADVERTISED_PORT=9092
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
- KAFKA_CREATE_TOPICS=kafka_docker_topic:1:1
ports:
- 9092:9092
depends_on:
- zookeeper
spring:
profiles: dev
cloud:
stream:
defaultBinder: kafka
kafka:
binder:
brokers: kafka # i added brokers and zkNodes property
zkNodes: zookeeper #
bindings:
input:
destination: message
content-type: application/json