JanusGraph not communicating with Cassandra backend in Docker environment - docker

I need your help on configuiring a docker compose file.
Problem : Created a custom build docker file ( janusgraph ) and trying to make it talk to other containers ( cassandra ) but it doesn't as it shows the below error..
janusgraph-es | 2449 [main] ERROR org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor - Could not invoke constructor on class org.janusgraph.graphdb.management.JanusGraphManager (defined by the 'graphManager' setting) with one argument of class Settings
janusgraph-es | Exception in thread "main" java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
janusgraph-es | at org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor.<init>(ServerGremlinExecutor.java:91)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.GremlinServer.<init>(GremlinServer.java:122)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.GremlinServer.<init>(GremlinServer.java:86)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.GremlinServer.main(GremlinServer.java:345)
janusgraph-es | Caused by: java.lang.reflect.InvocationTargetException
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
janusgraph-es | at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
janusgraph-es | at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor.<init>(ServerGremlinExecutor.java:80)
janusgraph-es | ... 3 more
janusgraph-es | Caused by: java.lang.IllegalArgumentException: Could not instantiate implementation: org.janusgraph.diskstorage.cql.CQLStoreManager
janusgraph-es | at org.janusgraph.util.system.ConfigurationUtil.instantiate(ConfigurationUtil.java:64)
janusgraph-es | at org.janusgraph.diskstorage.Backend.getImplementationClass(Backend.java:440)
janusgraph-es | at org.janusgraph.diskstorage.Backend.getStorageManager(Backend.java:411)
janusgraph-es | at org.janusgraph.graphdb.configuration.builder.GraphDatabaseConfigurationBuilder.build(GraphDatabaseConfigurationBuilder.java:50)
janusgraph-es | at org.janusgraph.core.JanusGraphFactory.lambda$open$0(JanusGraphFactory.java:150)
janusgraph-es | at org.janusgraph.graphdb.management.JanusGraphManager.openGraph(JanusGraphManager.java:243)
janusgraph-es | at org.janusgraph.core.JanusGraphFactory.open(JanusGraphFactory.java:150)
janusgraph-es | at org.janusgraph.core.JanusGraphFactory.open(JanusGraphFactory.java:100)
janusgraph-es | at org.janusgraph.graphdb.management.JanusGraphManager.lambda$new$0(JanusGraphManager.java:75)
janusgraph-es | at java.util.LinkedHashMap.forEach(LinkedHashMap.java:684)
janusgraph-es | at org.janusgraph.graphdb.management.JanusGraphManager.<init>(JanusGraphManager.java:74)
janusgraph-es | ... 8 more
janusgraph-es | Caused by: java.lang.reflect.InvocationTargetException
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
janusgraph-es | at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
janusgraph-es | at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
janusgraph-es | at org.janusgraph.util.system.ConfigurationUtil.instantiate(ConfigurationUtil.java:58)
janusgraph-es | ... 18 more
janusgraph-es | Caused by: com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: /127.0.0.1:9042 (com.datastax.driver.core.exceptions.TransportException: [/127.0.0.1:9042] Cannot connect))
janusgraph-es | at com.datastax.driver.core.ControlConnection.reconnectInternal(ControlConnection.java:268)
janusgraph-es | at com.datastax.driver.core.ControlConnection.connect(ControlConnection.java:107)
janusgraph-es | at com.datastax.driver.core.Cluster$Manager.negotiateProtocolVersionAndConnect(Cluster.java:1813)
janusgraph-es | at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1726)
janusgraph-es | at com.datastax.driver.core.Cluster.init(Cluster.java:214)
janusgraph-es | at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:387)
janusgraph-es | at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:366)
janusgraph-es | at com.datastax.driver.core.Cluster.connect(Cluster.java:311)
janusgraph-es | at org.janusgraph.diskstorage.cql.CQLStoreManager.initializeSession(CQLStoreManager.java:320)
janusgraph-es | at org.janusgraph.diskstorage.cql.CQLStoreManager.<init>(CQLStoreManager.java:182)
janusgraph-es | ... 23 more
Sharing the Dockerfile and the docker compose file
Dockerfile
FROM openjdk:8-jre-slim-buster
USER root
RUN mkdir -p /janusgraph
COPY . /janusgraph/.
WORKDIR /janusgraph/bin/
CMD ["./gremlin-server.sh"]
docker compose
version: "3.3"
services:
cassandra:
image: cassandra:3
container_name: cassandra
ports:
- "9042:9042"
- "9160:9160"
healthcheck:
test: ["CMD-SHELL", "[ $$(nodetool statusgossip) = running ]"]
interval: 30s
timeout: 10s
networks:
- jce-network
volumes:
- app-volume:/var/lib/cassandra
janusgraph-es:
# image: docker.io/janusgraph/janusgraph:0.5.3
build: ./janusgraph-full-0.5.3
container_name: janusgraph-es
# environment:
# JANUS_PROPS_TEMPLATE: cql-es
# janusgraph.storage.hostname: cassandra
# janusgraph.index.search.hostname: elastic
# graph.replace-instance-if-exists: true
ports:
- "8182:8182"
healthcheck:
test: ["CMD", "bin/gremlin.sh", "-e", "scripts/remote-connect.groovy"]
interval: 10000s
timeout: 3000s
retries: 3
links:
- "cassandra"
networks:
- jce-network
networks:
jce-network:
volumes:
app-volume:
FYI : I have edited the config of janusgraph by giving cassandra_container ip and also localhost and also if I start janusgraph manually with cassandra as docker service, it works!!
But not able to connect with using janusgraph as a docker service to cassandra.

The NoHostAvailableException in the stacktrace indicates that the Cassandra Java driver used by JanusGraph can't connect to the cluster:
...
Caused by: com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: /127.0.0.1:9042 (com.datastax.driver.core.exceptions.TransportException: [/127.0.0.1:9042] Cannot connect))
at com.datastax.driver.core.ControlConnection.reconnectInternal(ControlConnection.java:268)
at com.datastax.driver.core.ControlConnection.connect(ControlConnection.java:107)
at com.datastax.driver.core.Cluster$Manager.negotiateProtocolVersionAndConnect(Cluster.java:1813)
at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1726)
at com.datastax.driver.core.Cluster.init(Cluster.java:214)
...
I suspect that the problem is JanusGraph attempts to connect to the cluster before Cassandra is up and ready.
Try configuring your JanusGraph service to have a dependency on your Cassandra service (with depends_on). This means that the DB will be started before the app. Additionally, require that Cassandra is "healthy" before creating the JanusGraph service with the condition property.
Your Docker compose file should look like:
...
janusgraph-es:
...
depends_on:
cassandra:
condition: service_healthy
...
For details, see the official Docker Docs depends_on specification.
As a side note, container links is a legacy feature and is deprecated. Consider using the default bridge in your network configuration. Cheers!
networks:
jce-network:
driver: bridge
👉 Please support the Apache Cassandra community by hovering over the cassandra tag then click on the Watch tag button. 🙏 Thanks!

Related

Caused by: java.net.UnknownHostException: alc-vbox-testarchiveservice.minio with Minio and docker-compose

We are using Minio for local testing of S3 AND we have created docker-compose file with Minio and our app dependency are as follows:
Docker-Compose File:
version: "2.1"
services:
minio:
image: minio/minio
container_name: minio
ports:
- 9001:9001
volumes:
- minio_storage:/data
environment:
MINIO_ACCESS_KEY: minio
MINIO_SECRET_KEY: minio123
MINIO_REGION: us-east-1
command: server /data --console-address ":9001"
mem_limit: 512m
populate-minio-data:
container_name: "minio-data"
image: minio/mc
volumes:
- ./hello.txt:/tmp/hello.txt
entrypoint: >
/bin/sh -c "
/usr/bin/mc config host rm local;
/usr/bin/mc config host add --quiet --api s3v4 local http://minio:9001 minio minio123;
/usr/bin/mc mb --quiet local/somebucketname1/;
/usr/bin/mc policy set public local/somebucketname1;
/usr/bin/mc cp /tmp/hello.txt local/somebucketname1/hello.txt;
"
depends_on:
- minio
archive-api-app:
image: openjdk:11
container_name: "archive-api-app"
ports:
- 8091:6001
volumes:
- /home/apcuser/dev/projects/ea-archive-service-v2/projects/application/archive-api:/app
command: [ 'java', '-jar', '/app/build/libs/archive-api-1.0.0.jar' ]
env_file:
- ./vars/default.env
volumes:
minio_storage:
And In java code, I have configured MINIO URL as S3 Endpoint as follows:
#Bean
public AmazonS3 getS3Client() {
return AmazonS3ClientBuilder.standard()
.withClientConfiguration(new ClientConfiguration().withMaxConnections(maxConnections)
.withConnectionTimeout(connectionTimeout).withMaxErrorRetry(maxRetry))
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://minio:9001", "us-east-1"))
.build();
}
Once, I run docker-compose file,I am able to see minio-ui in my local Linux machine as follows:
But when I am hitting API request from postman to my archive-api-app, App throwing below error:
archive-api | at java.base/java.lang.Thread.run(Thread.java:829)
archive-api | Caused by: java.net.UnknownHostException: alc-vbox-testarchiveservice.minio
archive-api | at java.base/java.net.InetAddress$CachedAddresses.get(InetAddress.java:797)
archive-api | at java.base/java.net.InetAddress.getAllByName0(InetAddress.java:1519)
archive-api | at java.base/java.net.InetAddress.getAllByName(InetAddress.java:1378)
archive-api | at java.base/java.net.InetAddress.getAllByName(InetAddress.java:1306)
archive-api | at com.amazonaws.SystemDefaultDnsResolver.resolve(SystemDefaultDnsResolver.java:27)
archive-api | at com.amazonaws.http.DelegatingDnsResolver.resolve(DelegatingDnsResolver.java:38)
archive-api | at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:112)
archive-api | at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:376)
archive-api | at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
archive-api | at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
archive-api | at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
archive-api | at java.base/java.lang.reflect.Method.invoke(Method.java:566)
archive-api | at com.amazonaws.http.conn.ClientConnectionManagerFactory$Handler.invoke(ClientConnectionManagerFactory.java:76)
archive-api | at com.amazonaws.http.conn.$Proxy97.connect(Unknown Source)
archive-api | at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393)
archive-api | at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
archive-api | at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
archive-api | at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
archive-api | at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
archive-api | at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
archive-api | at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72)
archive-api | at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1346)
archive-api | at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1157)
archive-api | ... 77 more
Does anyone know Why I am getting UnknownHostException? My doubt is since, I configured Minio-url(http://minio:9001) in the code where I am creating S3 configs, then why my code is trying to connect to alc-vbox-testarchiveservice.minio host? (My bucket name is: alc-vbox-testarchiveservice)

docker rabbit mq image fails to load up

i have a project that i am using rabbitmq image docker when i use this command in my project :
docker-compose -f dc.rabbitmq.yml up
i get the below error for failing to start the container :
477a2217c16b: Pull complete
Digest: sha256:d8fb3795026b4c81eae33f4990e8bbc7b29c877388eef6ead2aca2945074c3f3
Status: Downloaded newer image for rabbitmq:3-management-alpine
Creating rabbitmq ... done
Attaching to rabbitmq
rabbitmq | Configuring logger redirection
rabbitmq | 07:50:26.108 [error]
rabbitmq |
rabbitmq | BOOT FAILED
rabbitmq | 07:50:26.116 [error] BOOT FAILED
rabbitmq | 07:50:26.117 [error] ===========
rabbitmq | ===========
rabbitmq | 07:50:26.117 [error] Exception during startup:
rabbitmq | 07:50:26.117 [error]
rabbitmq | Exception during startup:
rabbitmq |
rabbitmq | supervisor:'-start_children/2-fun-0-'/3 line 355
rabbitmq | 07:50:26.117 [error] supervisor:'-start_children/2-fun-0-'/3 line 355
rabbitmq | 07:50:26.117 [error] supervisor:do_start_child/2 line 371
rabbitmq | 07:50:26.117 [error] supervisor:do_start_child_i/3 line 385
rabbitmq | supervisor:do_start_child/2 line 371
rabbitmq | supervisor:do_start_child_i/3 line 385
rabbitmq | rabbit_prelaunch:run_prelaunch_first_phase/0 line 27
rabbitmq | rabbit_prelaunch:do_run/0 line 108
rabbitmq | 07:50:26.117 [error] rabbit_prelaunch:run_prelaunch_first_phase/0 line 27
rabbitmq | 07:50:26.117 [error] rabbit_prelaunch:do_run/0 line 108
rabbitmq | 07:50:26.117 [error] rabbit_prelaunch_conf:setup/1 line 33
rabbitmq | 07:50:26.117 [error] rabbit_prelaunch_conf:decrypt_config/2 line 404
rabbitmq | rabbit_prelaunch_conf:setup/1 line 33
rabbitmq | rabbit_prelaunch_conf:decrypt_config/2 line 404
rabbitmq | rabbit_prelaunch_conf:decrypt_app/3 line 425
rabbitmq | 07:50:26.117 [error] rabbit_prelaunch_conf:decrypt_app/3 line 425
rabbitmq | 07:50:26.117 [error] throw:{config_decryption_error,{key,default_pass},badarg}
rabbitmq | throw:{config_decryption_error,{key,default_pass},badarg}
rabbitmq |
rabbitmq | 07:50:26.117 [error]
rabbitmq | 07:50:27.118 [error] Supervisor rabbit_prelaunch_sup had child prelaunch started with rabbit_prelaunch:run_prelaunch_first_phase() at undefined exit with reason {config_decryption_error,{key,default_pass},badarg} in context start_error
so what i have tried so far what trying to search for any child proccess who fails the rabbit but could not find any and removed images and container and build and up them again but again the same error .
#EDIT
here is the config file :
version: "3.7"
services:
rabbitmq:
image: rabbitmq:3-management-alpine
container_name: rabbitmq
ports:
- "15672:15672"
- "5672:5672"
volumes:
- ./docker/rabbitmq/definitions.json:/opt/definitions.json:ro
- ./docker/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.config:ro
- dms_rabbitmq_data:/var/lib/rabbitmq/
- dms_rabbitmq_logs:/var/log/rabbitmq
networks:
- rabbitmq_network
labels:
- co.elastic.logs/module=rabbitmq
- co.elastic.logs/fileset.stdout=access
- co.elastic.logs/fileset.stderr=error
- co.elastic.metrics/module=rabbitmq
- co.elastic.metrics/metricsets=status
volumes:
dms_rabbitmq_etc:
name: dms_rabbitmq_etc
dms_rabbitmq_data:
name: dms_rabbitmq_data
dms_rabbitmq_logs:
driver: local
driver_opts:
type: "none"
o: "bind"
device: ${PWD}/storage/logs/rabbitmq
networks:
rabbitmq_network:
name: rabbitmq_network
I had the same problem. The solution for me was recreating an encoded password.

error restarting JBoss/Keycloak docker container

I am setting up a keycloak server using the Jboss docker image(v.4.0.0.final) and I am encountering this error upon reboot:
.....
keycloak_1 | at org.jboss.as.controller.AbstractControllerService$1.run(AbstractControllerService.java:370)
keycloak_1 | at java.lang.Thread.run(Thread.java:748)
keycloak_1 |
keycloak_1 | 18:51:15,509 ERROR [org.jboss.as.controller.client] (Controller Boot Thread) WFLYCTL0190: Step handler org.jboss.as.server.DeployerChainAddHandler$FinalRuntimeStepHandler#52734583 for operation add-deployer-chains at address [] failed handling operation rollback -- java.util.concurrent.RejectedExecutionException: Task org.jboss.as.controller.notification.NotificationSupports$NonBlockingNotificationSupport$1#2211e00e rejected from java.util.concurrent.ThreadPoolExecutor#41133ff1[Shutting down, pool size = 50, active threads = 43, queued tasks = 0, completed tasks = 93]
keycloak_1 | docker_keycloak_1 exited with code 1
When I run the docker container for the first time, everything goes as planned. I can login, create roles, etc. I launch the container using jhipster's keycloak script:
version: '2'
services:
keycloak:
image: jboss/keycloak:4.0.0.Final
command: ["-b", "0.0.0.0", "-Dkeycloak.migration.action=import", "-Dkeycloak.migration.provider=dir", "-Dkeycloak.migration.dir=/opt/jboss/keycloak/realm-config", "-Dkeycloak.migration.strategy=OVERWRITE_EXISTING", "-Djboss.socket.binding.port-offset=1000"]
volumes:
- ./realm-config:/opt/jboss/keycloak/realm-config
environment:
- KEYCLOAK_USER=admin
- KEYCLOAK_PASSWORD=admin
- DB_DATABASE=postgres
ports:
- 9080:9080
- 9443:9443
- 10990:10990
Am I shutting it down correctly?

Docker compose - Cannot execute request on any known server

I have the following issue. I have two spring boot micro services (1.5.8 ver), a configuration service and a discovery service.
When I run them locally the services are starting and communicating successfully.
Now when I start this two services using a docker compose file I get the following error:
Request execution failed with message: java.net.ConnectException: Connection refused (Connection refused)
discovery-svc | WARN 24 --- [nfoReplicator-0] com.netflix.discovery.DiscoveryClient : DiscoveryClient_DISCOVERY/6c4a9f118423:discovery - registration failed Cannot execute request on any known server
discovery-svc |
discovery-svc | com.netflix.discovery.shared.transport.TransportException: Cannot execute request on any known server
discovery-svc | at com.netflix.discovery.shared.transport.decorator.RetryableEurekaHttpClient.execute(RetryableEurekaHttpClient.java:111) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator$1.execute(EurekaHttpClientDecorator.java:59) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.SessionedEurekaHttpClient.execute(SessionedEurekaHttpClient.java:77) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.DiscoveryClient.register(DiscoveryClient.java:807) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.InstanceInfoReplicator.run(InstanceInfoReplicator.java:104) [eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_151]
discovery-svc | at java.lang.Thread.run(Thread.java:748) [na:1.8.0_151]
discovery-svc |
discovery-svc | 2017-12-20 15:19:00.314 WARN 24 --- [nfoReplicator-0] c.n.discovery.InstanceInfoReplicator : There was a problem with the instance info replicator
discovery-svc |
discovery-svc | com.netflix.discovery.shared.transport.TransportException: Cannot execute request on any known server
discovery-svc | at com.netflix.discovery.shared.transport.decorator.RetryableEurekaHttpClient.execute(RetryableEurekaHttpClient.java:111) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator$1.execute(EurekaHttpClientDecorator.java:59) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.SessionedEurekaHttpClient.execute(SessionedEurekaHttpClient.java:77) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.DiscoveryClient.register(DiscoveryClient.java:807) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at com.netflix.discovery.InstanceInfoReplicator.run(InstanceInfoReplicator.java:104) ~[eureka-client-1.7.0.jar!/:1.7.0]
discovery-svc | at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_151]
discovery-svc | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_151]
discovery-svc | at java.lang.Thread.run(Thread.java:748) [na:1.8.0_151]
I have the following files:
docker-compose.yml
version: '3.4'
services:
config-svc:
image: "config-img"
container_name: config-svc
build:
context: ../config
dockerfile: docker/config.dockerfile
expose:
- "8888"
discovery-svc:
image: "discovery-img"
container_name: discovery-svc
build:
context: ../discovery
dockerfile: docker/discovery.dockerfile
depends_on:
- config-svc
links:
- config-svc:config-svc
entrypoint: sh -c '/tmp/wait-for config-svc:8888 && java -jar discovery.jar'
expose:
- "8761"
Discovery service: bootstrap.yml
spring:
application:
name: discovery
cloud:
config:
uri: http://config-svc:8888
profiles:
active: docker
management:
security:
enabled: false
Configuration service: bootstrap.yml
server:
port: 8888
spring:
application:
name: config
cloud:
config:
server:
native:
search-locations: ${user.dir}/src/main/resources/configs
profiles:
active:
- native
security:
basic:
enabled: false
management:
security:
enabled: false
discovery.docker.properties
server.port=8761
eureka.client.register-with-eureka=false
eureka.client.fetch-registry=false
eureka.client.serviceUrl.defaultZone=http://discovery-svc:8761/eureka/
eureka.instance.hostname=discovery-svc
eureka.instance.preferIpAddress=false
logging.level.com.netflix.eureka=OFF
logging.level.com.netflix.discovery=OFF
I have tried many things but with no luck.
Can anyone suggest anything?
Thanks in advance
are you has been try set network each service ?
so, service config-svc and service discovery-svc communication through same network. try setup driver to bridge as follow
version: '3.4'
services:
config-svc:
image: "config-img"
container_name: config-svc
build:
context: ../config
dockerfile: docker/config.dockerfile
expose:
- "8888"
networks:
- app-network
discovery-svc:
image: "discovery-img"
container_name: discovery-svc
build:
context: ../discovery
dockerfile: docker/discovery.dockerfile
depends_on:
- config-svc
links:
- config-svc:config-svc
entrypoint: sh -c '/tmp/wait-for config-svc:8888 && java -jar discovery.jar'
expose:
- "8761"
networks:
- app-network
networks:
app-network:
driver: brige
reference: https://docs.docker.com/compose/compose-file/
Ok I found it. The problem was in the configuration service in the bootstrap.yml file the property value of spring.cloud.config.server.native.search-locations must be fixed to classpath:configs. Now it's working fine.
Thanks for your reply

unable to run kibana and logstash with elasticsearch

Elastic serach is running fine on 9201 port. But unable to run kibana and logstash with docker-compose.
For logstash it throws the error:
Attempted to resurrect connection to dead ES instance, but got an
error.
For kibana it throw warnings:
"warning","elasticsearch","admin"],"pid":1,"message":"No living
connections"
Below is the docker-compose.yml file:
version: '2'
services:
# Service 1 : elasticsearch
elasticsearch-5-6:
image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3
container_name: elasticsearch-5-6
ports:
- "9201:9200"
volumes:
- /etc/elasticsearch/elasticsearch-5-6.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- /var/elasticsearch/data/immunedata-5-6/:/usr/share/elasticsearch/data/
#- /etc/elasticsearch/logging.yml:/usr/share/elasticsearch/config/logging.yml
#- /var/log/elasticsearch/:/usr/share/elasticsearch/logs/
environment:
- cluster.name=docker-cluster-elasticsearch-5-6
#- bootstrap.memory_lock=true
- "ES_JAVA_OPTS: -Xmx2048m -Xms2048m"
# Disabling the xpack security as it costs after one month of free trail.
- xpack.security.enabled=false
# Service 2 : logstash
logstash-5-6:
image: docker.elastic.co/logstash/logstash:5.6.3
container_name: logstash-5-6
ports:
#- "5044:5044"
- "5001:5001"
volumes:
- /etc/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
- /etc/logstash/pipeline:/usr/share/logstash/pipeline
#- /etc/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
#- /var/logstash/pipeline:/usr/share/logstash/pipeline
environment:
- "ES_JAVA_OPTS: -Xmx2048m -Xms2048m"
depends_on:
- elasticsearch-5-6
# Service 3 : kibana
kibana-5-6:
image: docker.elastic.co/kibana/kibana:5.6.3
container_name: kibana-5-6
ports:
- "5601:5601"
volumes:
- /etc/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
#- /var/kibana/immunedata-5-6/:/usr/share/kibana/data/
environment:
- xpack.security.enabled=false
- xpack.graph.enabled = false
- xpack.ml.enabled = false
- xpack.monitoring.enabled = false
- xpack.watcher.enabled = false
- xpack.reporting.enabled = false
depends_on:
- elasticsearch-5-6
# Service 4 : elasticseach-head
elasticsearch-head:
image: mobz/elasticsearch-head:5
container_name: elasticsearch-head
# will not wait for elasticsearch to be ready.
ports:
- "9100:9100"
elasticserach.yml
cluster.name: immunedata-cluster-5.6
node.name: "immunedata-cluster-5-6.node-1"
# Elasticsearch in docker access different data directory, defined mapping directory in docker-compose.yml
#path.data: /var/elasticsearch/data/immunedata-5-6/
path.data: /usr/share/elasticsearch/data/
#path.data: /var/elasticsearch/data
# NOTE : Since elasticsearch 5.x index level settings can NOT be set on the nodes configuration like the elasticsearch.yaml
#index.number_of_shards: 1
#index.number_of_replicas: 0
# Allow all host access
network.bind_host: 0.0.0.0
http.port: 9200
# To enable cross-origin resource sharing (Accessing on browser)
http.cors.enabled: true
http.cors.allow-origin : "*"
logstash.yml file
http.host: "0.0.0.0"
path.config: /usr/share/logstash/pipeline
#xpack.monitoring.elasticsearch.url: http://localhost:9201
##xpack.monitoring.elasticsearch.url: http://elasticsearch:9201
#xpack.monitoring.elasticsearch.username: logstash_system
#xpack.monitoring.elasticsearch.password: changeme
xpack.monitoring.enabled: false
kibana.yml file
server.name: kibana
server.host: "0"
elasticsearch.url: http://192.168.56.10:9201
xpack.monitoring.ui.container.elasticsearch.enabled: false
#elasticsearch.url: http://elasticsearch:9201
xpack.security.enabled: false
## Above I tired this - not working
#elasticsearch.username: elastic
#elasticsearch.password: changeme
#xpack.monitoring.ui.container.elasticsearch.enabled: false
#xpack.monitoring.ui.container.elasticsearch.enabled: true
# Extra:
ssl.verificationMode: false
Logs:
[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_141]
elasticsearch-5-6 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_141]
elasticsearch-5-6 | at java.lang.Thread.run(Thread.java:748) [?:1.8.0_141]
elasticsearch-5-6 | [2017-11-26T06:07:57,084][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][young][14][6] duration [18.2s], collections [1]/[18.5s], total [18.2s]/[23.5s], memory [178.2mb]->[79.5mb]/[1.9gb], all_pools {[young] [132.1mb]->[964kb]/[133.1mb]}{[survivor] [16.6mb]->[12.5mb]/[16.6mb]}{[old] [29.4mb]->[66.5mb]/[1.8gb]}
elasticsearch-5-6 | [2017-11-26T06:07:57,085][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][14] overhead, spent [18.2s] collecting in the last [18.5s]
elasticsearch-5-6 | [2017-11-26T06:07:57,298][ERROR][o.e.x.m.c.i.IndexRecoveryCollector] [immunedata-cluster-5-6.node-1] collector [index-recovery] failed to collect data
elasticsearch-5-6 | org.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];
elasticsearch-5-6 | at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:165) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction.checkGlobalBlock(TransportRecoveryAction.java:114) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction.checkGlobalBlock(TransportRecoveryAction.java:52) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.<init>(TransportBroadcastByNodeAction.java:256) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction.doExecute(TransportBroadcastByNodeAction.java:234) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction.doExecute(TransportBroadcastByNodeAction.java:79) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:170) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:142) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:84) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) ~[elasticsearch-5.6.3.jar:5.6.3]
elasticsearch-5-6 | at
elasticsearch-5-6 | [2017-11-26T06:08:45,238][WARN ][o.e.x.w.e.ExecutionService] [immunedata-cluster-5-6.node-1] Failed to execute watch [XYNCje-TQzKm9OLdiH60gQ_elasticsearch_cluster_status_60e3c208-acca-4462-ba47-0711279d8f5e-2017-11-26T06:08:35.573Z]
elasticsearch-5-6 | [2017-11-26T06:08:54,886][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][young][63][9] duration [3.6s], collections [1]/[4.6s], total [3.6s]/[30.2s], memory [226.9mb]->[103.5mb]/[1.9gb], all_pools {[young] [127.5mb]->[1mb]/[133.1mb]}{[survivor] [16.6mb]->[11.3mb]/[16.6mb]}{[old] [82.7mb]->[91.2mb]/[1.8gb]}
elasticsearch-5-6 | [2017-11-26T06:08:54,886][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][63] overhead, spent [3.6s] collecting in the last [4.6s]
logstash-5-6 | Sending Logstash's logs to /usr/share/logstash/logs which is now configured via log4j2.properties
elasticsearch-5-6 | [2017-11-26T06:08:55,988][INFO ][o.e.c.r.a.AllocationService] [immunedata-cluster-5-6.node-1] Cluster health status changed from [RED] to [YELLOW] (reason: [shards started [[.watcher-history-6-2017.11.20][0], [.monitoring-es-6-2017.11.20][0]] ...]).
logstash-5-6 | [2017-11-26T06:08:56,786][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
logstash-5-6 | [2017-11-26T06:08:56,891][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"}
logstash-5-6 | [2017-11-26T06:08:57,558][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"arcsight", :directory=>"/usr/share/logstash/vendor/bundle/jruby/1.9/gems/x-pack-5.6.3-java/modules/arcsight/configuration"}
logstash-5-6 | [2017-11-26T06:09:04,121][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elastic:xxxxxx#elasticsearch-5-6:9201/]}}
logstash-5-6 | [2017-11-26T06:09:04,123][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
elasticsearch-5-6 | [2017-11-26T06:09:04,687][WARN ][o.e.c.r.a.DiskThresholdMonitor] [immunedata-cluster-5-6.node-1] high disk watermark [90%] exceeded on [eAlcHBJ2QVG58e0HJsgrdQ][immunedata-cluster-5-6.node-1][/usr/share/elasticsearch/data/nodes/0] free: 1.9gb[7.4%], shards will be relocated away from this node
elasticsearch-5-6 | [2017-11-26T06:09:04,687][INFO ][o.e.c.r.a.DiskThresholdMonitor] [immunedata-cluster-5-6.node-1] rerouting shards: [high disk watermark exceeded on one or more nodes]
logstash-5-6 | [2017-11-26T06:09:06,450][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
logstash-5-6 | [2017-11-26T06:09:06,452][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
logstash-5-6 | [2017-11-26T06:09:06,455][ERROR][logstash.outputs.elasticsearch] Failed to install template. {:message=>"Template file '' could not be found!", :class=>"ArgumentError", :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/template_manager.rb:37:in `read_template_file'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/template_manager.rb:23:in `get_template'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/template_manager.rb:7:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/common.rb:58:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/common.rb:25:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:9:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/output_delegator.rb:43:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:290:in `register_plugin'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:310:in `start_workers'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:235:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:398:in `start_pipeline'"]}
logstash-5-6 | [2017-11-26T06:09:06,455][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//elasticsearch-5-6:9201"]}
logstash-5-6 | [2017-11-26T06:09:06,462][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>250}
logstash-5-6 | [2017-11-26T06:09:09,818][INFO ][logstash.pipeline ] Pipeline main started
logstash-5-6 | [2017-11-26T06:09:10,341][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
logstash-5-6 | [2017-11-26T06:09:11,460][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
logstash-5-6 | [2017-11-26T06:09:11,484][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
logstash-5-6 | [2017-11-26T06:09:16,491][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
logstash-5-6 | [2017-11-26T06:09:16,500][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:21Z","tags":["warning","elasticsearch","config","deprecation"],"pid":1,"message":"Config key \"ssl.verify\" is deprecated. It has been replaced with \"ssl.verificationMode\""}
logstash-5-6 | [2017-11-26T06:09:21,513][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
logstash-5-6 | [2017-11-26T06:09:21,523][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:26Z","tags":["status","plugin:kibana#5.6.3","info"],"pid":1,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
logstash-5-6 | [2017-11-26T06:09:26,536][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
logstash-5-6 | [2017-11-26T06:09:26,570][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:26Z","tags":["status","plugin:elasticsearch#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:26Z","tags":["status","plugin:xpack_main#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["status","plugin:graph#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["error","elasticsearch","admin"],"pid":1,"message":"Request error, retrying\nHEAD http://elasticsearch-5-6:9201/ => connect ECONNREFUSED 172.21.0.2:9201"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["status","plugin:monitoring#5.6.3","info"],"pid":1,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"}
logstash-5-6 | [2017-11-26T06:09:31,585][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
logstash-5-6 | [2017-11-26T06:09:31,603][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["reporting","warning"],"pid":1,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent pending reports from failing on restart, please set xpack.reporting.encryptionKey in kibana.yml"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:reporting#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:xpack_main#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:graph#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:reporting#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:elasticsearch#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:searchprofiler#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from uninitialized to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"uninitialized","prevMsg":"uninitialized"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:34Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:34Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:34Z","tags":["status","plugin:ml#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from uninitialized to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"uninitialized","prevMsg":"uninitialized"}
elasticsearch-5-6 | [2017-11-26T06:09:34,750][WARN ][o.e.c.r.a.DiskThresholdMonitor] [immunedata-cluster-5-6.node-1] high disk watermark [90%] exceeded on [eAlcHBJ2QVG58e0HJsgrdQ][immunedata-cluster-5-6.node-1][/usr/share/elasticsearch/data/nodes/0] free: 1.9gb[7.4%], shards will be relocated away from this node
logstash-5-6 | [2017-11-26T06:09:36,692][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["status","plugin:ml#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from red to yellow - Waiting for Elasticsearch","prevState":"red","prevMsg":"Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201."}
logstash-5-6 | [2017-11-26T06:09:37,366][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"}
kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":
You called elasticsearch service elasticsearch-5-6 in your docker-compose.yml. That means that container with elasticsearch is available on address http://elasticsearch-5-6:9200 for all other containers in your docker-compose.yaml. And it is available on address http://127.0.0.1:9201 from the host machine.
In order to have workable ELK stack you need to change logstash config to:
http.host: "0.0.0.0"
path.config: /usr/share/logstash/pipeline
xpack.monitoring.elasticsearch.url: http://elasticsearch-5-6:9200
#xpack.monitoring.elasticsearch.username: logstash_system
#xpack.monitoring.elasticsearch.password: changeme
xpack.monitoring.enabled: false
and kibana config to:
server.name: kibana
server.host: "0"
elasticsearch.url: http://elasticsearch-5-6:9200
xpack.monitoring.ui.container.elasticsearch.enabled: false
xpack.security.enabled: false
## Above I tired this - not working
#elasticsearch.username: elastic
#elasticsearch.password: changeme
#xpack.monitoring.ui.container.elasticsearch.enabled: false
#xpack.monitoring.ui.container.elasticsearch.enabled: true
# Extra:
ssl.verificationMode: false
EKL Cluster with Xpack disabled
You are missing with the ELASTICSEARCH_URL: "http://elasticsearch:9200" in kibana and xpack.monitoring.elasticsearch.url: http://elasticsearch:9200 in Logstash
here is the sample yml configuration ith all possible environment varibales defined in environment
version: '3.4'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0
container_name: elasticsearch
environment:
ES_JAVA_OPTS: '-Xms2048m -Xmx2048m'
cluster.name: es-cluster
node.name: es1
network.bind_host: 0.0.0.0
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts: elasticsearch1
xpack.security.enabled: 'false'
xpack.monitoring.enabled: 'false'
xpack.watcher.enabled: 'false'
xpack.ml.enabled: 'false'
http.cors.enabled : 'true'
http.cors.allow-origin : "*"
http.cors.allow-methods : OPTIONS, HEAD, GET, POST, PUT, DELETE
http.cors.allow-headers : X-Requested-With,X-Auth-Token,Content-Type, Content-Length
logger.level: debug
volumes:
- /var/elasticsearch/db/elasticsearch/data:/usr/share/elasticsearch/data
ports:
- 9200:9200
- 9300:9300
networks:
- elastic
logstash:
image: docker.elastic.co/logstash/logstash:6.6.0
container_name: logstash
ports:
- 5044:5044
- 5001:5001
volumes:
- /var/elasticsearch/logstash/pipeline:/usr/share/logstash/pipeline
environment:
ES_JAVA_OPTS: -Xmx2048m -Xms2048m"
http.host: 0.0.0.0
xpack.monitoring.enabled: 'false'
xpack.monitoring.elasticsearch.url: http://elasticsearch:9200
networks:
- elastic
depends_on:
- elasticsearch
kibana:
image: docker.elastic.co/kibana/kibana:6.6.0
container_name: kibana
environment:
ELASTICSEARCH_URL: "http://elasticsearch:9200"
xpack.security.enabled: 'false'
xpack.graph.enabled : 'false'
xpack.ml.enabled : 'false'
xpack.monitoring.enabled : 'false'
xpack.watcher.enabled : 'false'
xpack.reporting.enabled : 'false'
ports:
- 5601:5601
networks:
- elastic
depends_on:
- elasticsearch
elasticsearch-head:
image: mobz/elasticsearch-head:5
container_name: elasticsearch-head
ports:
- "9100:9100"
networks:
- elastic
networks:
elastic:
driver: bridge

Resources