Can't access docker from LAN - docker

Question

I am running docker on my physic fedora machine.I started a prometheus container with host network, I think I should be able to access prometheus web pages from other host in the same local network, but failed.
My machina's ip is 172.31.209.112, the host I want to access prometheus is 172.31.1.195 on which I can ping 172.31.209.112. I can also confirm firewall is not running on both sides.
To access the LAN network I need to install a security software, which only has windows version, so I installed a windows vm with network set to NAT to cheat the security software, but I don't think this have something to do with my docker accessing problem.
I can access prometheus with localhost:9090 or 172.31.209.112:9090 locally, but just can't access it from any hosts in LAN, please help.
here is my docker container config:
{
"Id": "88b4c38e6a659754e861976b6b8b11d1dff495db1ef3d572169065a2e0acf4f6",
"Created": "2019-07-03T10:06:59.61293669Z",
"Path": "/bin/prometheus",
"Args": [
"--config.file=/etc/prometheus/prometheus.yml",
"--storage.tsdb.path=/prometheus",
"--web.console.libraries=/usr/share/prometheus/console_libraries",
"--web.console.templates=/usr/share/prometheus/consoles"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 5687,
"ExitCode": 0,
"Error": "",
"StartedAt": "2019-07-03T10:06:59.836329996Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:5d62a6125e7e0121151c24f6ec132bfae02cc37a2b57a666fd0569bed66d498f",
"ResolvConfPath": "/var/lib/docker/containers/88b4c38e6a659754e861976b6b8b11d1dff495db1ef3d572169065a2e0acf4f6/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/88b4c38e6a659754e861976b6b8b11d1dff495db1ef3d572169065a2e0acf4f6/hostname",
"HostsPath": "/var/lib/docker/containers/88b4c38e6a659754e861976b6b8b11d1dff495db1ef3d572169065a2e0acf4f6/hosts",
"LogPath": "/var/lib/docker/containers/88b4c38e6a659754e861976b6b8b11d1dff495db1ef3d572169065a2e0acf4f6/88b4c38e6a659754e861976b6b8b11d1dff495db1ef3d572169065a2e0acf4f6-json.log",
"Name": "/elastic_kapitsa",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/home/ggfan/4-tmp/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "host",
"PortBindings": {},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "shareable",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/d1e89e203e0a0791e370911f07f112a56603be51af65c29fc679b0c78350d4bf-init/diff:/var/lib/docker/overlay2/64e3d0a748213274b8210bde9f849e09e40aa59214cec724f0ef3167ed0d8030/diff:/var/lib/docker/overlay2/197e352b2464399aef2da8a75f31cb78179c3ad7b7e03b517176694fca19e7bc/diff:/var/lib/docker/overlay2/2a601b39acac1f8054c28af2d68004611ee17417c82d9c12842fc1b4725d18f5/diff:/var/lib/docker/overlay2/3590491c1df9eb9d1f4272abad88a6ad44f7f7e169129b8a6bdbba8a02fca09f/diff:/var/lib/docker/overlay2/0b1a049f0c204896187f0a3050a43eeb7d226fd0bfa810149f689039215d0068/diff:/var/lib/docker/overlay2/8c3523f1a5d77616be86ca76850c2cb682728c9d349032c0809781e3a9f24839/diff:/var/lib/docker/overlay2/b95196ec6ef2ed89345e6ecda9c476a30266a14097ca3931169159e01d5610b8/diff:/var/lib/docker/overlay2/56f16ca07e36898774523cb9d17bb5fc4d9c6526579bc3d764d257c41970e5bb/diff:/var/lib/docker/overlay2/749405b668264446ef9ec29a24089b5ce7dc7b9363033afab5dfa765ad1980e6/diff",
"MergedDir": "/var/lib/docker/overlay2/d1e89e203e0a0791e370911f07f112a56603be51af65c29fc679b0c78350d4bf/merged",
"UpperDir": "/var/lib/docker/overlay2/d1e89e203e0a0791e370911f07f112a56603be51af65c29fc679b0c78350d4bf/diff",
"WorkDir": "/var/lib/docker/overlay2/d1e89e203e0a0791e370911f07f112a56603be51af65c29fc679b0c78350d4bf/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/home/ggfan/4-tmp/prometheus/prometheus.yml",
"Destination": "/etc/prometheus/prometheus.yml",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "volume",
"Name": "833a1e3a523e0e3061f937ef9868413342ba9277c079b8f3bab5d363a884aa44",
"Source": "/var/lib/docker/volumes/833a1e3a523e0e3061f937ef9868413342ba9277c079b8f3bab5d363a884aa44/_data",
"Destination": "/prometheus",
"Driver": "local",
"Mode": "",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "localhost.localdomain",
"Domainname": "",
"User": "nobody",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"9090/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"--config.file=/etc/prometheus/prometheus.yml",
"--storage.tsdb.path=/prometheus",
"--web.console.libraries=/usr/share/prometheus/console_libraries",
"--web.console.templates=/usr/share/prometheus/consoles"
],
"ArgsEscaped": true,
"Image": "prom/prometheus",
"Volumes": {
"/prometheus": {}
},
"WorkingDir": "/prometheus",
"Entrypoint": [
"/bin/prometheus"
],
"OnBuild": null,
"Labels": {
"maintainer": "The Prometheus Authors <prometheus-developers#googlegroups.com>"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "5029bdc13e1f15aebb36e2d5a84c6d34f2d078360705a577946f92268b59a37e",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "/var/run/docker/netns/default",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"host": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "d58749e2d0012b4e59f12097a590b4814258fad4a99252a21df1c2863f5ca03b",
"EndpointID": "7238756953fc7ea91d9364c17a7730658f2057b583508bd1ea8ebcbffd1f6ab5",
"Gateway": "",
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "",
"DriverOpts": null
}
}
}
}
my network:
docker0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 172.17.0.1 netmask 255.255.0.0 broadcast 172.17.255.255
inet6 fe80::42:10ff:fe57:b91 prefixlen 64 scopeid 0x20<link>
ether 02:42:10:57:0b:91 txqueuelen 0 (Ethernet)
RX packets 33469 bytes 5113347 (4.8 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 40202 bytes 4779129 (4.5 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
enp0s31f6: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 172.31.209.112 netmask 255.255.255.0 broadcast 172.31.209.255
inet6 fe80::7ab9:b4e3:e089:36e3 prefixlen 64 scopeid 0x20<link>
ether 18:66:da:45:1f:63 txqueuelen 1000 (Ethernet)
RX packets 2448716 bytes 386681103 (368.7 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 761559 bytes 192020136 (183.1 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
device interrupt 16 memory 0xf7180000-f71a0000
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1000 (Local Loopback)
RX packets 256361 bytes 332219233 (316.8 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 256361 bytes 332219233 (316.8 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
veth19c1c5e: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet6 fe80::e80f:ccff:fefb:f2a1 prefixlen 64 scopeid 0x20<link>
ether ea:0f:cc:fb:f2:a1 txqueuelen 0 (Ethernet)
RX packets 83 bytes 285072 (278.3 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 5896 bytes 1195103 (1.1 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
veth4586d02: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet6 fe80::24c6:88ff:fe1b:41ea prefixlen 64 scopeid 0x20<link>
ether 26:c6:88:1b:41:ea txqueuelen 0 (Ethernet)
RX packets 39 bytes 2718 (2.6 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 6957 bytes 1410747 (1.3 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 192.168.122.1 netmask 255.255.255.0 broadcast 192.168.122.255
ether 52:54:00:2c:67:b9 txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0-nic: flags=4098<BROADCAST,MULTICAST> mtu 1500
ether 52:54:00:2c:67:b9 txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
vmnet1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 169.254.131.1 netmask 255.255.0.0 broadcast 169.254.255.255
inet6 fe80::eb22:59d6:882e:9200 prefixlen 64 scopeid 0x20<link>
ether 00:50:56:c0:00:01 txqueuelen 1000 (Ethernet)
RX packets 14178 bytes 0 (0.0 B)
RX errors 0 dropped 9 overruns 0 frame 0
TX packets 6603 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
and my routes:
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
default _gateway 0.0.0.0 UG 100 0 0 enp0s31f6
link-local 0.0.0.0 255.255.0.0 U 101 0 0 vmnet1
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
172.31.0.0 _gateway 255.255.0.0 UG 100 0 0 enp0s31f6
172.31.209.0 0.0.0.0 255.255.255.0 U 100 0 0 enp0s31f6
192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0
224.0.0.0 0.0.0.0 240.0.0.0 U 101 0 0 vmnet1