Not resovalbe hostnames of RabbitMQ cluster when deployed on Docker Swarm using the replica parameter - docker

Question

I use Docker Swarm to deploy 3 instances of RabbitMQ and Consul for the peer discovery.
version: '3.7'
services:
rabbitmq-1:
image: rabbitmq:3.8.9-alpine
hostname: "rabbitmq-1"
environment:
- RABBITMQ_ERLANG_COOKIE="SECRET_COOKIE"
networks:
- prod-net
configs:
- source: rabbitmq-config
target: /etc/rabbitmq/rabbitmq.conf
- source: rabbitmq-plugins-config
target: /etc/rabbitmq/enabled_plugins
volumes:
- /shared/rabbitmq/1/config:/etc/rabbitmq
- /shared/rabbitmq/1/data:/var/lib/rabbitmq
deploy:
replicas: 1
rabbitmq-2:
image: rabbitmq:3.8.9-alpine
hostname: "rabbitmq-2"
environment:
- RABBITMQ_ERLANG_COOKIE="SECRET_COOKIE"
networks:
- prod-net
configs:
- source: rabbitmq-config
target: /etc/rabbitmq/rabbitmq.conf
- source: rabbitmq-plugins-config
target: /etc/rabbitmq/enabled_plugins
volumes:
- /shared/rabbitmq/2/config:/etc/rabbitmq
- /shared/rabbitmq/2/data:/var/lib/rabbitmq
deploy:
replicas: 1
rabbitmq-3:
image: rabbitmq:3.8.9-alpine
hostname: "rabbitmq-3"
environment:
- RABBITMQ_ERLANG_COOKIE="SECRET_COOKIE"
networks:
- prod-net
configs:
- source: rabbitmq-config
target: /etc/rabbitmq/rabbitmq.conf
- source: rabbitmq-plugins-config
target: /etc/rabbitmq/enabled_plugins
volumes:
- /shared/rabbitmq/3/config:/etc/rabbitmq
- /shared/rabbitmq/3/data:/var/lib/rabbitmq
deploy:
replicas: 1
networks:
consul-net:
external: true
name: prod-net
configs:
rabbitmq-config:
external: true
rabbitmq-plugins-config:
external: true
RabbitMQ registers itself with Consul by its hostname so that the nodes can discover each other. So all the hostnames must resolve on all nodes. In the example above I'm using the same value for both the hostname and the service name to achieve this. But I would like to use a more compact way to represent a cluster by using a replicas: 3 parameter.
version: '3.7'
services:
rabbitmq:
image: rabbitmq:3.8.9-alpine
hostname: "rabbitmq-{{.Task.Slot}}"
networks:
- prod-net
configs:
- source: rabbitmq-config
target: /etc/rabbitmq/rabbitmq.conf
- source: rabbitmq-plugins-config
target: /etc/rabbitmq/enabled_plugins
deploy:
replicas: 3
restart_policy:
condition: any
networks:
prod-net:
external: true
configs:
rabbitmq-config:
external: true
rabbitmq-plugins-config:
external: true
Sadly in this case the hostnames are not anymore resolvable by docker DNS and the nodes cannot see each other. It there a way to achieve this ?
Edit.1:
Using these in rabbitmq-config gives the errors bellow:
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_dns
cluster_formation.dns.hostname = tasks.rabbitmq
2020-10-15 22:06:01.573 [error] <0.272.0> attempted to contact: ['rabbit#rabbitmq-3','rabbit#rabbitmq-2']
2020-10-15 22:06:01.574 [error] <0.272.0> rabbit#rabbitmq-3:
2020-10-15 22:06:01.574 [error] <0.272.0> * unable to connect to epmd (port 4369) on rabbitmq-3: couldn't resolve hostname
2020-10-15 22:06:01.574 [error] <0.272.0> rabbit#rabbitmq-2:
2020-10-15 22:06:01.575 [error] <0.272.0> * unable to connect to epmd (port 4369) on rabbitmq-2: couldn't resolve hostname
Edit.2:
rabbitmq.conf using Consul peer discovery
# Credentials
default_user = admin
default_pass = Pa$$w0rd1
loopback_users.admin = false
vm_memory_high_watermark.absolute = 1024MiB
disk_free_limit.absolute = 5GB
loopback_users.guest = false
# TLS Support in RabbitMQ
listeners.ssl.default = 5671
# Disables non-TLS listeners, only TLS-enabled clients will be able to connect
listeners.tcp = none
ssl_options.cacertfile = /etc/rabbitmq/ca_certificate.pem
ssl_options.certfile = /etc/rabbitmq/server_certificate.pem
ssl_options.keyfile = /etc/rabbitmq/server_key.pem
ssl_options.password = Pa$$phr#se
ssl_options.verify = verify_peer
ssl_options.fail_if_no_peer_cert = true
# TLS Support in RabbitMQ UI
management.ssl.port = 15671
management.ssl.cacertfile = /etc/rabbitmq/ca_certificate.pem
management.ssl.certfile = /etc/rabbitmq/server_certificate.pem
management.ssl.keyfile = /etc/rabbitmq/server_key.pem
management.ssl.password = Pa$$phr#se
management.ssl.verify = verify_none
management.ssl.fail_if_no_peer_cert = false
management.ssl.honor_cipher_order = true
management.ssl.honor_ecc_order = true
management.ssl.client_renegotiation = false
management.ssl.secure_renegotiate = true
cluster_partition_handling = autoheal
# Consul peer discovery
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_consul
# Consul host (hostname or IP address)
cluster_formation.consul.host = consul_client
# Service name (as registered in Consul) defaults to "rabbitmq"
cluster_formation.consul.svc = rabbitmq
# Compute service address (if not specify it below)
cluster_formation.consul.svc_addr_auto = true
cluster_formation.consul.use_longname = true
cluster_formation.consul.svc_ttl = 50
cluster_formation.consul.deregister_after = 100
cluster_formation.node_cleanup.only_log_warning = true
Or using DNS peer discovery:
#DNS Peer Discovery
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_dns
cluster_formation.dns.hostname = tasks.rabbitmq

Answer 1

To achieve DNS resolve between RabbitMQ replicas, need to modify /etc/hosts of each RabbitMQ container.
This will allow to form RabbitMQ cluster via rabbtimq configuration file.
See demo on how to create RabbitMQ cluster using single docker swarm service. I removed references to Consul and TLS from rabbitmq.conf for simplicity.