Docker swarm worker nodes bad gateway - docker

Question

I setup Traefik with docker swarm based on this guide. It only worked for single node for the SSL. But I add certificate to my other nodes on swarm.
Update cert all the nodes
docker node update --label-add traefik-public.traefik-public-certificates=true $NODE_ID
Start Traefik as Manager Role
docker service create \
--name traefik \
--constraint=node.labels.traefik-public.traefik-public-certificates==true \
--constraint=node.role==manager \
--publish 80:80 \
--publish 443:443 \
--mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \
--mount type=volume,source=traefik-public-certificates,target=/certificates \
--network traefik-public \
--label "traefik.frontend.rule=Host:monitor.$DOMAINNAME" \
--label "traefik.enable=true" \
--label "traefik.port=8080" \
--label "traefik.tags=traefik-public" \
--label "traefik.docker.network=traefik-public" \
--label "traefik.redirectorservice.frontend.entryPoints=http" \
--label "traefik.redirectorservice.frontend.redirect.entryPoint=https" \
--label "traefik.webservice.frontend.entryPoints=https" \
--label "traefik.frontend.auth.basic.users=${HTTP_USERNAME}:${HASHED_PASSWORD}" \
traefik:v1.7 \
--docker \
--docker.swarmmode \
--docker.watch \
--docker.exposedbydefault=false \
--constraints=tag==traefik-public \
--entrypoints='Name:http Address::80' \
--entrypoints='Name:https Address::443 TLS' \
--acme \
--acme.email=$EMAIL \
--acme.storage=/certificates/acme.json \
--acme.entryPoint=https \
--acme.httpChallenge.entryPoint=http\
--acme.onhostrule=true \
--acme.acmelogging=true \
--logLevel=INFO \
--accessLog \
--api
Docker Node Inspect Worker Node
[
{
"ID": "nv15pwb6bie7nvz2yk9uiii10",
"Version": {
"Index": 39862
},
"CreatedAt": "2019-04-20T09:11:29.540161196Z",
"UpdatedAt": "2019-04-22T05:37:40.858645073Z",
"Spec": {
"Labels": {
"traefik-public.traefik-public-certificates": "true"
},
"Role": "worker",
"Availability": "active"
},
"Description": {
"Hostname": "node-1",
"Platform": {
"Architecture": "x86_64",
"OS": "linux"
},
"Resources": {
"NanoCPUs": 1000000000,
"MemoryBytes": 1040351232
},
"Engine": {
"EngineVersion": "18.09.5",
"Labels": {
"provider": "digitalocean"
},
"Plugins": [
{
"Type": "Log",
"Name": "awslogs"
},
{
"Type": "Log",
"Name": "fluentd"
},
{
"Type": "Log",
"Name": "gcplogs"
},
{
"Type": "Log",
"Name": "gelf"
},
{
"Type": "Log",
"Name": "journald"
},
{
"Type": "Log",
"Name": "json-file"
},
{
"Type": "Log",
"Name": "local"
},
{
"Type": "Log",
"Name": "logentries"
},
{
"Type": "Log",
"Name": "splunk"
},
{
"Type": "Log",
"Name": "syslog"
},
{
"Type": "Network",
"Name": "bridge"
},
{
"Type": "Network",
"Name": "host"
},
{
"Type": "Network",
"Name": "macvlan"
},
{
"Type": "Network",
"Name": "null"
},
{
"Type": "Network",
"Name": "overlay"
},
{
"Type": "Volume",
"Name": "local"
}
]
},
"TLSInfo": {
"TrustRoot": "-----BEGIN CERTIFICATE-----\nMIIBazCCARCgAwIBAgIUJaJpxCmObYclnve1gMoDgqpbHpYwCgYIKoZIzj0EAwIw\nEzERMA8GA1UEAxMIc3dhcm0tY2EwHhcNMTkwNDIwMDEwMDAwWhcNMzkwNDE1MDEw\nMDAwWjATMREwDwYDVQQDEwhzd2FybS1jYTBZMBMGByqGSM49AgEGCCqGSM49AwEH\nA0IABKZ3tuUh1fbvsBrRxCr/2QpK42UXKH114Y5xUNjCdoVL7sDNJnPqHhGasbXZ\ncuYSf4oFPXau1Euqyo/lHFcn0TqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB\nAf8EBTADAQH/MB0GA1UdDgQWBBQLUasL0NneSvfzCIyelreL3Zl8GDAKBggqhkjO\nPQQDAgNJADBGAiEAuPDayJHm8fZEf1yYzS6CtrY/XuRYZK8NuNfG8Xsqs9YCIQDI\nPs6g4c65XPS7Gn931JEC/Qi7Zlu+ccMHy+Eup5SHsQ==\n-----END CERTIFICATE-----\n",
"CertIssuerSubject": "MBMxETAPBgNVBAMTCHN3YXJtLWNh",
"CertIssuerPublicKey": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEpne25SHV9u+wGtHEKv/ZCkrjZRcofXXhjnFQ2MJ2hUvuwM0mc+oeEZqxtdly5hJ/igU9dq7US6rKj+UcVyfROg=="
}
},
"Status": {
"State": "ready",
"Addr": "worker-machine-ip"
}
}
]
Docker Node Inspect Manager Node
{
"ID": "fw4k9vgz7y3929i731o7fll7d",
"Version": {
"Index": 39771
},
"CreatedAt": "2019-04-20T01:04:39.695763427Z",
"UpdatedAt": "2019-04-22T05:06:58.875733526Z",
"Spec": {
"Labels": {
"traefik-public.traefik-public-certificates": "true"
},
"Role": "manager",
"Availability": "active"
},
"Description": {
"Hostname": "shijie-master",
"Platform": {
"Architecture": "x86_64",
"OS": "linux"
},
"Resources": {
"NanoCPUs": 1000000000,
"MemoryBytes": 2097283072
},
"Engine": {
"EngineVersion": "18.09.4",
"Plugins": [
{
"Type": "Log",
"Name": "awslogs"
},
{
"Type": "Log",
"Name": "fluentd"
},
{
"Type": "Log",
"Name": "gcplogs"
},
{
"Type": "Log",
"Name": "gelf"
},
{
"Type": "Log",
"Name": "journald"
},
{
"Type": "Log",
"Name": "json-file"
},
{
"Type": "Log",
"Name": "local"
},
{
"Type": "Log",
"Name": "logentries"
},
{
"Type": "Log",
"Name": "splunk"
},
{
"Type": "Log",
"Name": "syslog"
},
{
"Type": "Network",
"Name": "bridge"
},
{
"Type": "Network",
"Name": "host"
},
{
"Type": "Network",
"Name": "macvlan"
},
{
"Type": "Network",
"Name": "null"
},
{
"Type": "Network",
"Name": "overlay"
},
{
"Type": "Volume",
"Name": "local"
}
]
},
"TLSInfo": {
"TrustRoot": "-----BEGIN CERTIFICATE-----\nMIIBazCCARCgAwIBAgIUJaJpxCmObYclnve1gMoDgqpbHpYwCgYIKoZIzj0EAwIw\nEzERMA8GA1UEAxMIc3dhcm0tY2EwHhcNMTkwNDIwMDEwMDAwWhcNMzkwNDE1MDEw\nMDAwWjATMREwDwYDVQQDEwhzd2FybS1jYTBZMBMGByqGSM49AgEGCCqGSM49AwEH\nA0IABKZ3tuUh1fbvsBrRxCr/2QpK42UXKH114Y5xUNjCdoVL7sDNJnPqHhGasbXZ\ncuYSf4oFPXau1Euqyo/lHFcn0TqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB\nAf8EBTADAQH/MB0GA1UdDgQWBBQLUasL0NneSvfzCIyelreL3Zl8GDAKBggqhkjO\nPQQDAgNJADBGAiEAuPDayJHm8fZEf1yYzS6CtrY/XuRYZK8NuNfG8Xsqs9YCIQDI\nPs6g4c65XPS7Gn931JEC/Qi7Zlu+ccMHy+Eup5SHsQ==\n-----END CERTIFICATE-----\n",
"CertIssuerSubject": "MBMxETAPBgNVBAMTCHN3YXJtLWNh",
"CertIssuerPublicKey": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEpne25SHV9u+wGtHEKv/ZCkrjZRcofXXhjnFQ2MJ2hUvuwM0mc+oeEZqxtdly5hJ/igU9dq7US6rKj+UcVyfROg=="
}
},
"Status": {
"State": "ready",
"Addr": "manager-machine-ip"
},
"ManagerStatus": {
"Leader": true,
"Reachability": "reachable",
"Addr": "manager-machine-ip:2377"
}
}
]
Docker network inspect on traefik-public
[
{
"Name": "traefik-public",
"Id": "6655p8lsxjmhqhha3e3fbs5xz",
"Created": "2019-04-21T06:07:01.862111049Z",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "10.0.12.0/24",
"Gateway": "10.0.12.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"Containers": { .. },
"ConfigOnly": false,
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4109"
},
"Labels": {},
"Peers": [...]
}
]
Problem
Running services on manager node works fine, but worker nodes return Error 502 Bad Gateway