Docker compose - Cannot execute request on any known server - docker

I have the following issue. I have two spring boot micro services (1.5.8 ver), a configuration service and a discovery service.
When I run them locally the services are starting and communicating successfully.
Now when I start this two services using a docker compose file I get the following error:
Request execution failed with message: java.net.ConnectException: Connection refused (Connection refused)
discovery-svc | WARN 24 --- [nfoReplicator-0] com.netflix.discovery.DiscoveryClient : DiscoveryClient_DISCOVERY/6c4a9f118423:discovery - registration failed Cannot execute request on any known server
discovery-svc |
discovery-svc | com.netflix.discovery.shared.transport.TransportException: Cannot execute request on any known server
discovery-svc | at com.netflix.discovery.shared.transport.decorator.RetryableEurekaHttpClient.execute(RetryableEurekaHttpClient.java:111) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator$1.execute(EurekaHttpClientDecorator.java:59) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.SessionedEurekaHttpClient.execute(SessionedEurekaHttpClient.java:77) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.DiscoveryClient.register(DiscoveryClient.java:807) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.InstanceInfoReplicator.run(InstanceInfoReplicator.java:104) [eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_151]
discovery-svc | at java.lang.Thread.run(Thread.java:748) [na:1.8.0_151]
discovery-svc |
discovery-svc | 2017-12-20 15:19:00.314 WARN 24 --- [nfoReplicator-0] c.n.discovery.InstanceInfoReplicator : There was a problem with the instance info replicator
discovery-svc |
discovery-svc | com.netflix.discovery.shared.transport.TransportException: Cannot execute request on any known server
discovery-svc | at com.netflix.discovery.shared.transport.decorator.RetryableEurekaHttpClient.execute(RetryableEurekaHttpClient.java:111) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator$1.execute(EurekaHttpClientDecorator.java:59) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.SessionedEurekaHttpClient.execute(SessionedEurekaHttpClient.java:77) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.DiscoveryClient.register(DiscoveryClient.java:807) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.InstanceInfoReplicator.run(InstanceInfoReplicator.java:104) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_151]
discovery-svc | at java.lang.Thread.run(Thread.java:748) [na:1.8.0_151]
I have the following files:
docker-compose.yml
version: '3.4'
services:
config-svc:
image: "config-img"
container_name: config-svc
build:
context: ../config
dockerfile: docker/config.dockerfile
expose:
- "8888"
discovery-svc:
image: "discovery-img"
container_name: discovery-svc
build:
context: ../discovery
dockerfile: docker/discovery.dockerfile
depends_on:
- config-svc
links:
- config-svc:config-svc
entrypoint: sh -c '/tmp/wait-for config-svc:8888 && java -jar discovery.jar'
expose:
- "8761"
Discovery service: bootstrap.yml
spring:
application:
name: discovery
cloud:
config:
uri: http://config-svc:8888
profiles:
active: docker
management:
security:
enabled: false
Configuration service: bootstrap.yml
server:
port: 8888
spring:
application:
name: config
cloud:
config:
server:
native:
search-locations: ${user.dir}/src/main/resources/configs
profiles:
active:
- native
security:
basic:
enabled: false
management:
security:
enabled: false
discovery.docker.properties
server.port=8761
eureka.client.register-with-eureka=false
eureka.client.fetch-registry=false
eureka.client.serviceUrl.defaultZone=http://discovery-svc:8761/eureka/
eureka.instance.hostname=discovery-svc
eureka.instance.preferIpAddress=false
logging.level.com.netflix.eureka=OFF
logging.level.com.netflix.discovery=OFF
I have tried many things but with no luck.
Can anyone suggest anything?
Thanks in advance

are you has been try set network each service ?
so, service config-svc and service discovery-svc communication through same network. try setup driver to bridge as follow
version: '3.4'
services:
config-svc:
image: "config-img"
container_name: config-svc
build:
context: ../config
dockerfile: docker/config.dockerfile
expose:
- "8888"
networks:
- app-network
discovery-svc:
image: "discovery-img"
container_name: discovery-svc
build:
context: ../discovery
dockerfile: docker/discovery.dockerfile
depends_on:
- config-svc
links:
- config-svc:config-svc
entrypoint: sh -c '/tmp/wait-for config-svc:8888 && java -jar discovery.jar'
expose:
- "8761"
networks:
- app-network
networks:
app-network:
driver: brige
reference: https://docs.docker.com/compose/compose-file/

Ok I found it. The problem was in the configuration service in the bootstrap.yml file the property value of spring.cloud.config.server.native.search-locations must be fixed to classpath:configs. Now it's working fine.
Thanks for your reply

Related

JanusGraph not communicating with Cassandra backend in Docker environment

I need your help on configuiring a docker compose file.
Problem : Created a custom build docker file ( janusgraph ) and trying to make it talk to other containers ( cassandra ) but it doesn't as it shows the below error..
janusgraph-es | 2449 [main] ERROR org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor - Could not invoke constructor on class org.janusgraph.graphdb.management.JanusGraphManager (defined by the 'graphManager' setting) with one argument of class Settings
janusgraph-es | Exception in thread "main" java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
janusgraph-es | at org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor.<init>(ServerGremlinExecutor.java:91)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.GremlinServer.<init>(GremlinServer.java:122)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.GremlinServer.<init>(GremlinServer.java:86)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.GremlinServer.main(GremlinServer.java:345)
janusgraph-es | Caused by: java.lang.reflect.InvocationTargetException
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
janusgraph-es | at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
janusgraph-es | at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor.<init>(ServerGremlinExecutor.java:80)
janusgraph-es | ... 3 more
janusgraph-es | Caused by: java.lang.IllegalArgumentException: Could not instantiate implementation: org.janusgraph.diskstorage.cql.CQLStoreManager
janusgraph-es | at org.janusgraph.util.system.ConfigurationUtil.instantiate(ConfigurationUtil.java:64)
janusgraph-es | at org.janusgraph.diskstorage.Backend.getImplementationClass(Backend.java:440)
janusgraph-es | at org.janusgraph.diskstorage.Backend.getStorageManager(Backend.java:411)
janusgraph-es | at org.janusgraph.graphdb.configuration.builder.GraphDatabaseConfigurationBuilder.build(GraphDatabaseConfigurationBuilder.java:50)
janusgraph-es | at org.janusgraph.core.JanusGraphFactory.lambda$open$0(JanusGraphFactory.java:150)
janusgraph-es | at org.janusgraph.graphdb.management.JanusGraphManager.openGraph(JanusGraphManager.java:243)
janusgraph-es | at org.janusgraph.core.JanusGraphFactory.open(JanusGraphFactory.java:150)
janusgraph-es | at org.janusgraph.core.JanusGraphFactory.open(JanusGraphFactory.java:100)
janusgraph-es | at org.janusgraph.graphdb.management.JanusGraphManager.lambda$new$0(JanusGraphManager.java:75)
janusgraph-es | at java.util.LinkedHashMap.forEach(LinkedHashMap.java:684)
janusgraph-es | at org.janusgraph.graphdb.management.JanusGraphManager.<init>(JanusGraphManager.java:74)
janusgraph-es | ... 8 more
janusgraph-es | Caused by: java.lang.reflect.InvocationTargetException
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
janusgraph-es | at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
janusgraph-es | at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
janusgraph-es | at org.janusgraph.util.system.ConfigurationUtil.instantiate(ConfigurationUtil.java:58)
janusgraph-es | ... 18 more
janusgraph-es | Caused by: com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: /127.0.0.1:9042 (com.datastax.driver.core.exceptions.TransportException: [/127.0.0.1:9042] Cannot connect))
janusgraph-es | at com.datastax.driver.core.ControlConnection.reconnectInternal(ControlConnection.java:268)
janusgraph-es | at com.datastax.driver.core.ControlConnection.connect(ControlConnection.java:107)
janusgraph-es | at com.datastax.driver.core.Cluster$Manager.negotiateProtocolVersionAndConnect(Cluster.java:1813)
janusgraph-es | at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1726)
janusgraph-es | at com.datastax.driver.core.Cluster.init(Cluster.java:214)
janusgraph-es | at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:387)
janusgraph-es | at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:366)
janusgraph-es | at com.datastax.driver.core.Cluster.connect(Cluster.java:311)
janusgraph-es | at org.janusgraph.diskstorage.cql.CQLStoreManager.initializeSession(CQLStoreManager.java:320)
janusgraph-es | at org.janusgraph.diskstorage.cql.CQLStoreManager.<init>(CQLStoreManager.java:182)
janusgraph-es | ... 23 more
Sharing the Dockerfile and the docker compose file
Dockerfile
FROM openjdk:8-jre-slim-buster
USER root
RUN mkdir -p /janusgraph
COPY . /janusgraph/.
WORKDIR /janusgraph/bin/
CMD ["./gremlin-server.sh"]
docker compose
version: "3.3"
services:
cassandra:
image: cassandra:3
container_name: cassandra
ports:
- "9042:9042"
- "9160:9160"
healthcheck:
test: ["CMD-SHELL", "[ $$(nodetool statusgossip) = running ]"]
interval: 30s
timeout: 10s
networks:
- jce-network
volumes:
- app-volume:/var/lib/cassandra
janusgraph-es:
# image: docker.io/janusgraph/janusgraph:0.5.3
build: ./janusgraph-full-0.5.3
container_name: janusgraph-es
# environment:
# JANUS_PROPS_TEMPLATE: cql-es
# janusgraph.storage.hostname: cassandra
# janusgraph.index.search.hostname: elastic
# graph.replace-instance-if-exists: true
ports:
- "8182:8182"
healthcheck:
test: ["CMD", "bin/gremlin.sh", "-e", "scripts/remote-connect.groovy"]
interval: 10000s
timeout: 3000s
retries: 3
links:
- "cassandra"
networks:
- jce-network
networks:
jce-network:
volumes:
app-volume:
FYI : I have edited the config of janusgraph by giving cassandra_container ip and also localhost and also if I start janusgraph manually with cassandra as docker service, it works!!
But not able to connect with using janusgraph as a docker service to cassandra.
The NoHostAvailableException in the stacktrace indicates that the Cassandra Java driver used by JanusGraph can't connect to the cluster:
...
Caused by: com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: /127.0.0.1:9042 (com.datastax.driver.core.exceptions.TransportException: [/127.0.0.1:9042] Cannot connect))
at com.datastax.driver.core.ControlConnection.reconnectInternal(ControlConnection.java:268)
at com.datastax.driver.core.ControlConnection.connect(ControlConnection.java:107)
at com.datastax.driver.core.Cluster$Manager.negotiateProtocolVersionAndConnect(Cluster.java:1813)
at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1726)
at com.datastax.driver.core.Cluster.init(Cluster.java:214)
...
I suspect that the problem is JanusGraph attempts to connect to the cluster before Cassandra is up and ready.
Try configuring your JanusGraph service to have a dependency on your Cassandra service (with depends_on). This means that the DB will be started before the app. Additionally, require that Cassandra is "healthy" before creating the JanusGraph service with the condition property.
Your Docker compose file should look like:
...
janusgraph-es:
...
depends_on:
cassandra:
condition: service_healthy
...
For details, see the official Docker Docs depends_on specification.
As a side note, container links is a legacy feature and is deprecated. Consider using the default bridge in your network configuration. Cheers!
networks:
jce-network:
driver: bridge
👉 Please support the Apache Cassandra community by hovering over the cassandra tag then click on the Watch tag button. 🙏 Thanks!

Caused by: java.net.UnknownHostException: alc-vbox-testarchiveservice.minio with Minio and docker-compose

We are using Minio for local testing of S3 AND we have created docker-compose file with Minio and our app dependency are as follows:
Docker-Compose File:
version: "2.1"
services:
minio:
image: minio/minio
container_name: minio
ports:
- 9001:9001
volumes:
- minio_storage:/data
environment:
MINIO_ACCESS_KEY: minio
MINIO_SECRET_KEY: minio123
MINIO_REGION: us-east-1
command: server /data --console-address ":9001"
mem_limit: 512m
populate-minio-data:
container_name: "minio-data"
image: minio/mc
volumes:
- ./hello.txt:/tmp/hello.txt
entrypoint: >
/bin/sh -c "
/usr/bin/mc config host rm local;
/usr/bin/mc config host add --quiet --api s3v4 local http://minio:9001 minio minio123;
/usr/bin/mc mb --quiet local/somebucketname1/;
/usr/bin/mc policy set public local/somebucketname1;
/usr/bin/mc cp /tmp/hello.txt local/somebucketname1/hello.txt;
"
depends_on:
- minio
archive-api-app:
image: openjdk:11
container_name: "archive-api-app"
ports:
- 8091:6001
volumes:
- /home/apcuser/dev/projects/ea-archive-service-v2/projects/application/archive-api:/app
command: [ 'java', '-jar', '/app/build/libs/archive-api-1.0.0.jar' ]
env_file:
- ./vars/default.env
volumes:
minio_storage:
And In java code, I have configured MINIO URL as S3 Endpoint as follows:
#Bean
public AmazonS3 getS3Client() {
return AmazonS3ClientBuilder.standard()
.withClientConfiguration(new ClientConfiguration().withMaxConnections(maxConnections)
.withConnectionTimeout(connectionTimeout).withMaxErrorRetry(maxRetry))
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://minio:9001", "us-east-1"))
.build();
}
Once, I run docker-compose file,I am able to see minio-ui in my local Linux machine as follows:
But when I am hitting API request from postman to my archive-api-app, App throwing below error:
archive-api | at java.base/java.lang.Thread.run(Thread.java:829)
archive-api | Caused by: java.net.UnknownHostException: alc-vbox-testarchiveservice.minio
archive-api | at java.base/java.net.InetAddress$CachedAddresses.get(InetAddress.java:797)
archive-api | at java.base/java.net.InetAddress.getAllByName0(InetAddress.java:1519)
archive-api | at java.base/java.net.InetAddress.getAllByName(InetAddress.java:1378)
archive-api | at java.base/java.net.InetAddress.getAllByName(InetAddress.java:1306)
archive-api | at com.amazonaws.SystemDefaultDnsResolver.resolve(SystemDefaultDnsResolver.java:27)
archive-api | at com.amazonaws.http.DelegatingDnsResolver.resolve(DelegatingDnsResolver.java:38)
archive-api | at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:112)
archive-api | at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:376)
archive-api | at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
archive-api | at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
archive-api | at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
archive-api | at java.base/java.lang.reflect.Method.invoke(Method.java:566)
archive-api | at com.amazonaws.http.conn.ClientConnectionManagerFactory$Handler.invoke(ClientConnectionManagerFactory.java:76)
archive-api | at com.amazonaws.http.conn.$Proxy97.connect(Unknown Source)
archive-api | at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393)
archive-api | at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
archive-api | at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
archive-api | at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
archive-api | at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
archive-api | at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
archive-api | at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72)
archive-api | at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1346)
archive-api | at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1157)
archive-api | ... 77 more
Does anyone know Why I am getting UnknownHostException? My doubt is since, I configured Minio-url(http://minio:9001) in the code where I am creating S3 configs, then why my code is trying to connect to alc-vbox-testarchiveservice.minio host? (My bucket name is: alc-vbox-testarchiveservice)

Elastic Search connection refused with Docker Compose (connect ECONNREFUSED )

There are multiple services that I had been trying to run (redis, front-end, back-end and elastic-search) and I was not able to connect to the elastic search service. I even tried giving a static ip for the service. (The networking part is currently commented out in the docker file attached). I tried changing the images and it still was not working.
When I tested ES locally using curl localhost:9200/_cat/health as I have mapped the container port locally it gives me that the cluster is green. I could connect to the other services like redis without issues. As with redis, I am using the service name, elasticsearch to connect it to the back-end service. Following is my docker-compose.yml file.
version: '3'
services:
arc-external:
image: arc-external
build:
context: ./arc-development-branch/arc-external
ports:
- '4201:4201'
# networks:
# - vpcbr
redis:
image: redis:3.2.11-alpine
ports:
- '6379:6379'
# networks:
# - vpcbr
elasticsearch:
image: elasticsearch:2
ports:
- '9200:9200'
- '9300:9300'
environment:
- node.name=elasticsearch
- cluster.name=datasearch
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- cluster.initial_master_nodes=elasticsearch
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- ./data/elastic:/usr/share/elasticsearch/data
# networks:
# vpcbr:
# ipv4_address: 10.5.0.4
api-external:
image: api-external
build: .
ports:
- '3001:3001'
depends_on:
- redis
- elasticsearch
# networks:
# - vpcbr
# networks:
# vpcbr:
# driver: bridge
# ipam:
# config:
# - subnet: 10.5.0.0/16
# gateway: 10.5.0.1
This is the exact error that I am getting when running docker compose-up
api-external_1 | 2021-03-09 20:41:46.3253 - info: Finished setting up log directories
api-external_1 | 2021-03-09 20:41:46.3514 - info: Connection successful to mongodb # mongodb://10.0.0.44:27017/arc
api-external_1 | 2021-03-09 20:41:46.3764 - info: Connection successful to redis at: host: redis port: 6379
api-external_1 | Elasticsearch ERROR: 2021-03-09T20:41:46Z
api-external_1 | Error: Request error, retrying
api-external_1 | HEAD http://elasticsearch:9200/ => connect ECONNREFUSED 172.24.0.4:9200
api-external_1 | at Log.error (/usr/src/app/api-external/node_modules/elasticsearch/src/lib/log.js:226:56)
api-external_1 | at checkRespForFailure (/usr/src/app/api-external/node_modules/elasticsearch/src/lib/transport.js:259:18)
api-external_1 | at HttpConnector.<anonymous> (/usr/src/app/api-external/node_modules/elasticsearch/src/lib/connectors/http.js:164:7)
api-external_1 | at ClientRequest.wrapper (/usr/src/app/api-external/node_modules/lodash/lodash.js:4935:19)
api-external_1 | at ClientRequest.emit (events.js:198:13)
api-external_1 | at ClientRequest.EventEmitter.emit (domain.js:448:20)
api-external_1 | at Socket.socketErrorListener (_http_client.js:401:9)
api-external_1 | at Socket.emit (events.js:198:13)
api-external_1 | at Socket.EventEmitter.emit (domain.js:448:20)
api-external_1 | at emitErrorNT (internal/streams/destroy.js:91:8)
api-external_1 | at emitErrorAndCloseNT (internal/streams/destroy.js:59:3)
api-external_1 | at process._tickCallback (internal/process/next_tick.js:63:19)
api-external_1 |
api-external_1 | Elasticsearch WARNING: 2021-03-09T20:41:46Z
api-external_1 | Unable to revive connection: http://elasticsearch:9200/
api-external_1 |
api-external_1 | Elasticsearch WARNING: 2021-03-09T20:41:46Z
api-external_1 | No living connections
api-external_1 |
api-external_1 | 2021-03-09 20:41:46.3844 - error: Error: Failed to connect to elasticsearch # elasticsearch:9200
api-external_1 | at exports.esClient.ping (/usr/src/app/api-external/dist/setup/elastic-search.js:33:46)
api-external_1 | at respond (/usr/src/app/api-external/node_modules/elasticsearch/src/lib/transport.js:327:9)
api-external_1 | at sendReqWithConnection (/usr/src/app/api-external/node_modules/elasticsearch/src/lib/transport.js:226:7)
api-external_1 | at next (/usr/src/app/api-external/node_modules/elasticsearch/src/lib/connection_pool.js:214:7)
api-external_1 | at process._tickCallback (internal/process/next_tick.js:61:11)
api-external_1 | 2021-03-09 20:41:46.3854 - error: Error: No Living connections
api-external_1 | at sendReqWithConnection (/usr/src/app/api-external/node_modules/elasticsearch/src/lib/transport.js:226:15)
api-external_1 | at next (/usr/src/app/api-external/node_modules/elasticsearch/src/lib/connection_pool.js:214:7)
api-external_1 | at process._tickCallback (internal/process/next_tick.js:61:11)
api-external_1 | npm ERR! code ELIFEC
Frankly, I searched a lot and were not able to debug it. Any help would be appreciated.
I was able to figure out the answer. The thing stating as depends_on does not wait the services to completely up. Here, api-external does get start up as soon as the redis and elasticsearch starts. However, elasticsearch need a bit time to configure everything so restarting the service will do the trick.
More permanent solution is to write a script that would wait until the elasticsearch is up completely before starting the api-external service

Starting WebAgent as a docker container

I am trying to deploy GridGain Web Console with Docker and also with web agent in docker-compose.yml file. I tried it according to this guide https://www.gridgain.com/docs/web-console/latest/deploying-web-console-docker, but web-agent is not part of docker deployment.
I tried to modify docker-compose.yml:
version: '2.4'
services:
backend:
image: gridgain/gridgain-web-console-backend:2020.05.00
# Restart on crash.
restart: always
environment:
# Mail settings
#- SPRING_MAIL_HOST=
#- SPRING_MAIL_PORT=
#- SPRING_MAIL_USERNAME=
#- SPRING_MAIL_PASSWORD=
- JVM_OPTS=
volumes:
- ${PWD}/Users:/opt/gridgain-web-console-server/work
frontend:
depends_on:
- backend
image: gridgain/gridgain-web-console-frontend:2020.05.00
ports:
# Proxy HTTP nginx port (HOST_PORT:DOCKER_PORT)
- 80:8008
container_name: frontend
webagent:
depends_on:
- frontend
- backend
image: gridgain/gridgain-web-agent:2020.05.00
environment:
- TOKENS=59f9d111-eaad-405b-a3c8-310e9245f943
- SERVER_URI=frontend:80
And it does not work. Always web agent is not able to connect to server(GridGain console). I run it via docker-compose up
I get following stacktrace:
webagent_1 | [2020-08-29T12:03:56,213][INFO ][Connect thread][WebSocketRouter] Connecting to server: frontend:80
webagent_1 | [2020-08-29T12:03:56,220][WARN ][Connect thread][AgentUtils] Failed to configure proxy.
webagent_1 | java.net.URISyntaxException: Expected scheme-specific part at index 6: https:
webagent_1 | at java.net.URI$Parser.fail(URI.java:2848) ~[?:1.8.0_242]
webagent_1 | at java.net.URI$Parser.failExpecting(URI.java:2854) ~[?:1.8.0_242]
webagent_1 | at java.net.URI$Parser.parse(URI.java:3057) ~[?:1.8.0_242]
webagent_1 | at java.net.URI.<init>(URI.java:673) ~[?:1.8.0_242]
webagent_1 | at org.gridgain.console.agent.AgentUtils.configureProxy(AgentUtils.java:156) ~[gridgain-web-console-agent-2020.05.00.jar:?]
webagent_1 | at org.gridgain.console.agent.handlers.WebSocketRouter.connect0(WebSocketRouter.java:225) ~[gridgain-web-console-agent-2020.05.00.jar:?]
webagent_1 | at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_242]
webagent_1 | at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_242]
webagent_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_242]
webagent_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_242]
webagent_1 | at java.lang.Thread.run(Thread.java:748) [?:1.8.0_242]
webagent_1 | [2020-08-29T12:03:57,032][ERROR][Connect thread][WebSocketRouter] Failed to establish websocket connection with server: frontend:80
webagent_1 | java.lang.IllegalArgumentException: URI scheme must be 'ws' or 'wss'
Question is how to correct connect webagent to gridgain console. I mean something important is missing in 'webagent' service in docker file, but I do not know what.
Have you any suggestion for resolving of my issue?
Thanks for everything.
You are using the wrong uri for the web console in the agent, please use SERVER_URI=http://frontend:8008
To get a token for agent you can only run web console docker-compose up -d backend frontend
register a user
open the profile page and copy the token
setting TOKENS in the web agent
startup agent docker-compose up -d webagent
But I can't find NODE_URI in your configuration file. By default agent will try to connect to http://localhost:8080, which is unreachable in web agent container.

unable to run kibana and logstash with elasticsearch

Elastic serach is running fine on 9201 port. But unable to run kibana and logstash with docker-compose.
For logstash it throws the error:
Attempted to resurrect connection to dead ES instance, but got an
error.
For kibana it throw warnings:
"warning","elasticsearch","admin"],"pid":1,"message":"No living
connections"
Below is the docker-compose.yml file:
version: '2'
services:
# Service 1 : elasticsearch
elasticsearch-5-6:
image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3
container_name: elasticsearch-5-6
ports:
- "9201:9200"
volumes:
- /etc/elasticsearch/elasticsearch-5-6.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- /var/elasticsearch/data/immunedata-5-6/:/usr/share/elasticsearch/data/
#- /etc/elasticsearch/logging.yml:/usr/share/elasticsearch/config/logging.yml
#- /var/log/elasticsearch/:/usr/share/elasticsearch/logs/
environment:
- cluster.name=docker-cluster-elasticsearch-5-6
#- bootstrap.memory_lock=true
- "ES_JAVA_OPTS: -Xmx2048m -Xms2048m"
# Disabling the xpack security as it costs after one month of free trail.
- xpack.security.enabled=false
# Service 2 : logstash
logstash-5-6:
image: docker.elastic.co/logstash/logstash:5.6.3
container_name: logstash-5-6
ports:
#- "5044:5044"
- "5001:5001"
volumes:
- /etc/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
- /etc/logstash/pipeline:/usr/share/logstash/pipeline
#- /etc/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
#- /var/logstash/pipeline:/usr/share/logstash/pipeline
environment:
- "ES_JAVA_OPTS: -Xmx2048m -Xms2048m"
depends_on:
- elasticsearch-5-6
# Service 3 : kibana
kibana-5-6:
image: docker.elastic.co/kibana/kibana:5.6.3
container_name: kibana-5-6
ports:
- "5601:5601"
volumes:
- /etc/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
#- /var/kibana/immunedata-5-6/:/usr/share/kibana/data/
environment:
- xpack.security.enabled=false
- xpack.graph.enabled = false
- xpack.ml.enabled = false
- xpack.monitoring.enabled = false
- xpack.watcher.enabled = false
- xpack.reporting.enabled = false
depends_on:
- elasticsearch-5-6
# Service 4 : elasticseach-head
elasticsearch-head:
image: mobz/elasticsearch-head:5
container_name: elasticsearch-head
# will not wait for elasticsearch to be ready.
ports:
- "9100:9100"
elasticserach.yml
cluster.name: immunedata-cluster-5.6
node.name: "immunedata-cluster-5-6.node-1"
# Elasticsearch in docker access different data directory, defined mapping directory in docker-compose.yml
#path.data: /var/elasticsearch/data/immunedata-5-6/
path.data: /usr/share/elasticsearch/data/
#path.data: /var/elasticsearch/data
# NOTE : Since elasticsearch 5.x index level settings can NOT be set on the nodes configuration like the elasticsearch.yaml
#index.number_of_shards: 1
#index.number_of_replicas: 0
# Allow all host access
network.bind_host: 0.0.0.0
http.port: 9200
# To enable cross-origin resource sharing (Accessing on browser)
http.cors.enabled: true
http.cors.allow-origin : "*"
logstash.yml file
http.host: "0.0.0.0"
path.config: /usr/share/logstash/pipeline
#xpack.monitoring.elasticsearch.url: http://localhost:9201
##xpack.monitoring.elasticsearch.url: http://elasticsearch:9201
#xpack.monitoring.elasticsearch.username: logstash_system
#xpack.monitoring.elasticsearch.password: changeme
xpack.monitoring.enabled: false
kibana.yml file
server.name: kibana
server.host: "0"
elasticsearch.url: http://192.168.56.10:9201
xpack.monitoring.ui.container.elasticsearch.enabled: false
#elasticsearch.url: http://elasticsearch:9201
xpack.security.enabled: false
## Above I tired this - not working
#elasticsearch.username: elastic
#elasticsearch.password: changeme
#xpack.monitoring.ui.container.elasticsearch.enabled: false
#xpack.monitoring.ui.container.elasticsearch.enabled: true
# Extra:
ssl.verificationMode: false
Logs:
[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_141]
elasticsearch-5-6 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_141]
elasticsearch-5-6 | at java.lang.Thread.run(Thread.java:748) [?:1.8.0_141]
elasticsearch-5-6 | [2017-11-26T06:07:57,084][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][young][14][6] duration [18.2s], collections [1]/[18.5s], total [18.2s]/[23.5s], memory [178.2mb]->[79.5mb]/[1.9gb], all_pools {[young] [132.1mb]->[964kb]/[133.1mb]}{[survivor] [16.6mb]->[12.5mb]/[16.6mb]}{[old] [29.4mb]->[66.5mb]/[1.8gb]}
elasticsearch-5-6 | [2017-11-26T06:07:57,085][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][14] overhead, spent [18.2s] collecting in the last [18.5s]
elasticsearch-5-6 | [2017-11-26T06:07:57,298][ERROR][o.e.x.m.c.i.IndexRecoveryCollector] [immunedata-cluster-5-6.node-1] collector [index-recovery] failed to collect data
elasticsearch-5-6 | org.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];
elasticsearch-5-6 | at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:165) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction.checkGlobalBlock(TransportRecoveryAction.java:114) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction.checkGlobalBlock(TransportRecoveryAction.java:52) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.<init>(TransportBroadcastByNodeAction.java:256) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction.doExecute(TransportBroadcastByNodeAction.java:234) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction.doExecute(TransportBroadcastByNodeAction.java:79) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:170) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:142) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:84) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at
elasticsearch-5-6 | [2017-11-26T06:08:45,238][WARN ][o.e.x.w.e.ExecutionService] [immunedata-cluster-5-6.node-1] Failed to execute watch [XYNCje-TQzKm9OLdiH60gQ_elasticsearch_cluster_status_60e3c208-acca-4462-ba47-0711279d8f5e-2017-11-26T06:08:35.573Z]
elasticsearch-5-6 | [2017-11-26T06:08:54,886][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][young][63][9] duration [3.6s], collections [1]/[4.6s], total [3.6s]/[30.2s], memory [226.9mb]->[103.5mb]/[1.9gb], all_pools {[young] [127.5mb]->[1mb]/[133.1mb]}{[survivor] [16.6mb]->[11.3mb]/[16.6mb]}{[old] [82.7mb]->[91.2mb]/[1.8gb]}
elasticsearch-5-6 | [2017-11-26T06:08:54,886][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][63] overhead, spent [3.6s] collecting in the last [4.6s]
logstash-5-6 | Sending Logstash's logs to /usr/share/logstash/logs which is now configured via log4j2.properties
elasticsearch-5-6 | [2017-11-26T06:08:55,988][INFO ][o.e.c.r.a.AllocationService] [immunedata-cluster-5-6.node-1] Cluster health status changed from [RED] to [YELLOW] (reason: [shards started [[.watcher-history-6-2017.11.20][0], [.monitoring-es-6-2017.11.20][0]] ...]).
logstash-5-6 | [2017-11-26T06:08:56,786][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
logstash-5-6 | [2017-11-26T06:08:56,891][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"}
logstash-5-6 | [2017-11-26T06:08:57,558][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"arcsight", :directory=>"/usr/share/logstash/vendor/bundle/jruby/1.9/gems/x-pack-5.6.3-java/modules/arcsight/configuration"}
logstash-5-6 | [2017-11-26T06:09:04,121][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elastic:xxxxxx#elasticsearch-5-6:9201/]}}
logstash-5-6 | [2017-11-26T06:09:04,123][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
elasticsearch-5-6 | [2017-11-26T06:09:04,687][WARN ][o.e.c.r.a.DiskThresholdMonitor] [immunedata-cluster-5-6.node-1] high disk watermark [90%] exceeded on [eAlcHBJ2QVG58e0HJsgrdQ][immunedata-cluster-5-6.node-1][/usr/share/elasticsearch/data/nodes/0] free: 1.9gb[7.4%], shards will be relocated away from this node
elasticsearch-5-6 | [2017-11-26T06:09:04,687][INFO ][o.e.c.r.a.DiskThresholdMonitor] [immunedata-cluster-5-6.node-1] rerouting shards: [high disk watermark exceeded on one or more nodes]
logstash-5-6 | [2017-11-26T06:09:06,450][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
logstash-5-6 | [2017-11-26T06:09:06,452][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
logstash-5-6 | [2017-11-26T06:09:06,455][ERROR][logstash.outputs.elasticsearch] Failed to install template. {:message=>"Template file '' could not be found!", :class=>"ArgumentError", :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/template_manager.rb:37:in `read_template_file'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/template_manager.rb:23:in `get_template'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/template_manager.rb:7:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/common.rb:58:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/common.rb:25:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:9:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/output_delegator.rb:43:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:290:in `register_plugin'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:310:in `start_workers'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:235:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:398:in `start_pipeline'"]}
logstash-5-6 | [2017-11-26T06:09:06,455][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//elasticsearch-5-6:9201"]}
logstash-5-6 | [2017-11-26T06:09:06,462][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>250}
logstash-5-6 | [2017-11-26T06:09:09,818][INFO ][logstash.pipeline ] Pipeline main started
logstash-5-6 | [2017-11-26T06:09:10,341][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
logstash-5-6 | [2017-11-26T06:09:11,460][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
logstash-5-6 | [2017-11-26T06:09:11,484][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
logstash-5-6 | [2017-11-26T06:09:16,491][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
logstash-5-6 | [2017-11-26T06:09:16,500][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:21Z","tags":["warning","elasticsearch","config","deprecation"],"pid":1,"message":"Config key \"ssl.verify\" is deprecated. It has been replaced with \"ssl.verificationMode\""}
logstash-5-6 | [2017-11-26T06:09:21,513][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
logstash-5-6 | [2017-11-26T06:09:21,523][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:26Z","tags":["status","plugin:kibana#5.6.3","info"],"pid":1,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
logstash-5-6 | [2017-11-26T06:09:26,536][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
logstash-5-6 | [2017-11-26T06:09:26,570][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:26Z","tags":["status","plugin:elasticsearch#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:26Z","tags":["status","plugin:xpack_main#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["status","plugin:graph#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["error","elasticsearch","admin"],"pid":1,"message":"Request error, retrying\nHEAD http://elasticsearch-5-6:9201/ => connect ECONNREFUSED 172.21.0.2:9201"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["status","plugin:monitoring#5.6.3","info"],"pid":1,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"}
logstash-5-6 | [2017-11-26T06:09:31,585][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
logstash-5-6 | [2017-11-26T06:09:31,603][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["reporting","warning"],"pid":1,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent pending reports from failing on restart, please set xpack.reporting.encryptionKey in kibana.yml"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:reporting#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:xpack_main#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:graph#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:reporting#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:elasticsearch#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:searchprofiler#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from uninitialized to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:34Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:34Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:34Z","tags":["status","plugin:ml#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from uninitialized to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"uninitialized","prevMsg":"uninitialized"}
elasticsearch-5-6 | [2017-11-26T06:09:34,750][WARN ][o.e.c.r.a.DiskThresholdMonitor] [immunedata-cluster-5-6.node-1] high disk watermark [90%] exceeded on [eAlcHBJ2QVG58e0HJsgrdQ][immunedata-cluster-5-6.node-1][/usr/share/elasticsearch/data/nodes/0] free: 1.9gb[7.4%], shards will be relocated away from this node
logstash-5-6 | [2017-11-26T06:09:36,692][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["status","plugin:ml#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from red to yellow - Waiting for Elasticsearch","prevState":"red","prevMsg":"Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201."}
logstash-5-6 | [2017-11-26T06:09:37,366][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":
You called elasticsearch service elasticsearch-5-6 in your docker-compose.yml. That means that container with elasticsearch is available on address http://elasticsearch-5-6:9200 for all other containers in your docker-compose.yaml. And it is available on address http://127.0.0.1:9201 from the host machine.
In order to have workable ELK stack you need to change logstash config to:
http.host: "0.0.0.0"
path.config: /usr/share/logstash/pipeline
xpack.monitoring.elasticsearch.url: http://elasticsearch-5-6:9200
#xpack.monitoring.elasticsearch.username: logstash_system
#xpack.monitoring.elasticsearch.password: changeme
xpack.monitoring.enabled: false
and kibana config to:
server.name: kibana
server.host: "0"
elasticsearch.url: http://elasticsearch-5-6:9200
xpack.monitoring.ui.container.elasticsearch.enabled: false
xpack.security.enabled: false
## Above I tired this - not working
#elasticsearch.username: elastic
#elasticsearch.password: changeme
#xpack.monitoring.ui.container.elasticsearch.enabled: false
#xpack.monitoring.ui.container.elasticsearch.enabled: true
# Extra:
ssl.verificationMode: false
EKL Cluster with Xpack disabled
You are missing with the ELASTICSEARCH_URL: "http://elasticsearch:9200" in kibana and xpack.monitoring.elasticsearch.url: http://elasticsearch:9200 in Logstash
here is the sample yml configuration ith all possible environment varibales defined in environment
version: '3.4'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0
container_name: elasticsearch
environment:
ES_JAVA_OPTS: '-Xms2048m -Xmx2048m'
cluster.name: es-cluster
node.name: es1
network.bind_host: 0.0.0.0
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts: elasticsearch1
xpack.security.enabled: 'false'
xpack.monitoring.enabled: 'false'
xpack.watcher.enabled: 'false'
xpack.ml.enabled: 'false'
http.cors.enabled : 'true'
http.cors.allow-origin : "*"
http.cors.allow-methods : OPTIONS, HEAD, GET, POST, PUT, DELETE
http.cors.allow-headers : X-Requested-With,X-Auth-Token,Content-Type, Content-Length
logger.level: debug
volumes:
- /var/elasticsearch/db/elasticsearch/data:/usr/share/elasticsearch/data
ports:
- 9200:9200
- 9300:9300
networks:
- elastic
logstash:
image: docker.elastic.co/logstash/logstash:6.6.0
container_name: logstash
ports:
- 5044:5044
- 5001:5001
volumes:
- /var/elasticsearch/logstash/pipeline:/usr/share/logstash/pipeline
environment:
ES_JAVA_OPTS: -Xmx2048m -Xms2048m"
http.host: 0.0.0.0
xpack.monitoring.enabled: 'false'
xpack.monitoring.elasticsearch.url: http://elasticsearch:9200
networks:
- elastic
depends_on:
- elasticsearch
kibana:
image: docker.elastic.co/kibana/kibana:6.6.0
container_name: kibana
environment:
ELASTICSEARCH_URL: "http://elasticsearch:9200"
xpack.security.enabled: 'false'
xpack.graph.enabled : 'false'
xpack.ml.enabled : 'false'
xpack.monitoring.enabled : 'false'
xpack.watcher.enabled : 'false'
xpack.reporting.enabled : 'false'
ports:
- 5601:5601
networks:
- elastic
depends_on:
- elasticsearch
elasticsearch-head:
image: mobz/elasticsearch-head:5
container_name: elasticsearch-head
ports:
- "9100:9100"
networks:
- elastic
networks:
elastic:
driver: bridge

Resources