JanusGraph not communicating with Cassandra backend in Docker environment - docker
I need your help on configuiring a docker compose file.
Problem : Created a custom build docker file ( janusgraph ) and trying to make it talk to other containers ( cassandra ) but it doesn't as it shows the below error..
janusgraph-es | 2449 [main] ERROR org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor - Could not invoke constructor on class org.janusgraph.graphdb.management.JanusGraphManager (defined by the 'graphManager' setting) with one argument of class Settings
janusgraph-es | Exception in thread "main" java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
janusgraph-es | at org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor.<init>(ServerGremlinExecutor.java:91)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.GremlinServer.<init>(GremlinServer.java:122)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.GremlinServer.<init>(GremlinServer.java:86)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.GremlinServer.main(GremlinServer.java:345)
janusgraph-es | Caused by: java.lang.reflect.InvocationTargetException
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
janusgraph-es | at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
janusgraph-es | at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
janusgraph-es | at org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor.<init>(ServerGremlinExecutor.java:80)
janusgraph-es | ... 3 more
janusgraph-es | Caused by: java.lang.IllegalArgumentException: Could not instantiate implementation: org.janusgraph.diskstorage.cql.CQLStoreManager
janusgraph-es | at org.janusgraph.util.system.ConfigurationUtil.instantiate(ConfigurationUtil.java:64)
janusgraph-es | at org.janusgraph.diskstorage.Backend.getImplementationClass(Backend.java:440)
janusgraph-es | at org.janusgraph.diskstorage.Backend.getStorageManager(Backend.java:411)
janusgraph-es | at org.janusgraph.graphdb.configuration.builder.GraphDatabaseConfigurationBuilder.build(GraphDatabaseConfigurationBuilder.java:50)
janusgraph-es | at org.janusgraph.core.JanusGraphFactory.lambda$open$0(JanusGraphFactory.java:150)
janusgraph-es | at org.janusgraph.graphdb.management.JanusGraphManager.openGraph(JanusGraphManager.java:243)
janusgraph-es | at org.janusgraph.core.JanusGraphFactory.open(JanusGraphFactory.java:150)
janusgraph-es | at org.janusgraph.core.JanusGraphFactory.open(JanusGraphFactory.java:100)
janusgraph-es | at org.janusgraph.graphdb.management.JanusGraphManager.lambda$new$0(JanusGraphManager.java:75)
janusgraph-es | at java.util.LinkedHashMap.forEach(LinkedHashMap.java:684)
janusgraph-es | at org.janusgraph.graphdb.management.JanusGraphManager.<init>(JanusGraphManager.java:74)
janusgraph-es | ... 8 more
janusgraph-es | Caused by: java.lang.reflect.InvocationTargetException
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
janusgraph-es | at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
janusgraph-es | at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
janusgraph-es | at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
janusgraph-es | at org.janusgraph.util.system.ConfigurationUtil.instantiate(ConfigurationUtil.java:58)
janusgraph-es | ... 18 more
janusgraph-es | Caused by: com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: /127.0.0.1:9042 (com.datastax.driver.core.exceptions.TransportException: [/127.0.0.1:9042] Cannot connect))
janusgraph-es | at com.datastax.driver.core.ControlConnection.reconnectInternal(ControlConnection.java:268)
janusgraph-es | at com.datastax.driver.core.ControlConnection.connect(ControlConnection.java:107)
janusgraph-es | at com.datastax.driver.core.Cluster$Manager.negotiateProtocolVersionAndConnect(Cluster.java:1813)
janusgraph-es | at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1726)
janusgraph-es | at com.datastax.driver.core.Cluster.init(Cluster.java:214)
janusgraph-es | at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:387)
janusgraph-es | at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:366)
janusgraph-es | at com.datastax.driver.core.Cluster.connect(Cluster.java:311)
janusgraph-es | at org.janusgraph.diskstorage.cql.CQLStoreManager.initializeSession(CQLStoreManager.java:320)
janusgraph-es | at org.janusgraph.diskstorage.cql.CQLStoreManager.<init>(CQLStoreManager.java:182)
janusgraph-es | ... 23 more
Sharing the Dockerfile and the docker compose file
Dockerfile
FROM openjdk:8-jre-slim-buster
USER root
RUN mkdir -p /janusgraph
COPY . /janusgraph/.
WORKDIR /janusgraph/bin/
CMD ["./gremlin-server.sh"]
docker compose
version: "3.3"
services:
cassandra:
image: cassandra:3
container_name: cassandra
ports:
- "9042:9042"
- "9160:9160"
healthcheck:
test: ["CMD-SHELL", "[ $$(nodetool statusgossip) = running ]"]
interval: 30s
timeout: 10s
networks:
- jce-network
volumes:
- app-volume:/var/lib/cassandra
janusgraph-es:
# image: docker.io/janusgraph/janusgraph:0.5.3
build: ./janusgraph-full-0.5.3
container_name: janusgraph-es
# environment:
# JANUS_PROPS_TEMPLATE: cql-es
# janusgraph.storage.hostname: cassandra
# janusgraph.index.search.hostname: elastic
# graph.replace-instance-if-exists: true
ports:
- "8182:8182"
healthcheck:
test: ["CMD", "bin/gremlin.sh", "-e", "scripts/remote-connect.groovy"]
interval: 10000s
timeout: 3000s
retries: 3
links:
- "cassandra"
networks:
- jce-network
networks:
jce-network:
volumes:
app-volume:
FYI : I have edited the config of janusgraph by giving cassandra_container ip and also localhost and also if I start janusgraph manually with cassandra as docker service, it works!!
But not able to connect with using janusgraph as a docker service to cassandra.
The NoHostAvailableException in the stacktrace indicates that the Cassandra Java driver used by JanusGraph can't connect to the cluster:
...
Caused by: com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: /127.0.0.1:9042 (com.datastax.driver.core.exceptions.TransportException: [/127.0.0.1:9042] Cannot connect))
at com.datastax.driver.core.ControlConnection.reconnectInternal(ControlConnection.java:268)
at com.datastax.driver.core.ControlConnection.connect(ControlConnection.java:107)
at com.datastax.driver.core.Cluster$Manager.negotiateProtocolVersionAndConnect(Cluster.java:1813)
at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1726)
at com.datastax.driver.core.Cluster.init(Cluster.java:214)
...
I suspect that the problem is JanusGraph attempts to connect to the cluster before Cassandra is up and ready.
Try configuring your JanusGraph service to have a dependency on your Cassandra service (with depends_on). This means that the DB will be started before the app. Additionally, require that Cassandra is "healthy" before creating the JanusGraph service with the condition property.
Your Docker compose file should look like:
...
janusgraph-es:
...
depends_on:
cassandra:
condition: service_healthy
...
For details, see the official Docker Docs depends_on specification.
As a side note, container links is a legacy feature and is deprecated. Consider using the default bridge in your network configuration. Cheers!
networks:
jce-network:
driver: bridge
👉 Please support the Apache Cassandra community by hovering over the cassandra tag then click on the Watch tag button. 🙏 Thanks!
Related
Caused by: java.net.UnknownHostException: alc-vbox-testarchiveservice.minio with Minio and docker-compose
We are using Minio for local testing of S3 AND we have created docker-compose file with Minio and our app dependency are as follows: Docker-Compose File: version: "2.1" services: minio: image: minio/minio container_name: minio ports: - 9001:9001 volumes: - minio_storage:/data environment: MINIO_ACCESS_KEY: minio MINIO_SECRET_KEY: minio123 MINIO_REGION: us-east-1 command: server /data --console-address ":9001" mem_limit: 512m populate-minio-data: container_name: "minio-data" image: minio/mc volumes: - ./hello.txt:/tmp/hello.txt entrypoint: > /bin/sh -c " /usr/bin/mc config host rm local; /usr/bin/mc config host add --quiet --api s3v4 local http://minio:9001 minio minio123; /usr/bin/mc mb --quiet local/somebucketname1/; /usr/bin/mc policy set public local/somebucketname1; /usr/bin/mc cp /tmp/hello.txt local/somebucketname1/hello.txt; " depends_on: - minio archive-api-app: image: openjdk:11 container_name: "archive-api-app" ports: - 8091:6001 volumes: - /home/apcuser/dev/projects/ea-archive-service-v2/projects/application/archive-api:/app command: [ 'java', '-jar', '/app/build/libs/archive-api-1.0.0.jar' ] env_file: - ./vars/default.env volumes: minio_storage: And In java code, I have configured MINIO URL as S3 Endpoint as follows: #Bean public AmazonS3 getS3Client() { return AmazonS3ClientBuilder.standard() .withClientConfiguration(new ClientConfiguration().withMaxConnections(maxConnections) .withConnectionTimeout(connectionTimeout).withMaxErrorRetry(maxRetry)) .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://minio:9001", "us-east-1")) .build(); } Once, I run docker-compose file,I am able to see minio-ui in my local Linux machine as follows: But when I am hitting API request from postman to my archive-api-app, App throwing below error: archive-api | at java.base/java.lang.Thread.run(Thread.java:829) archive-api | Caused by: java.net.UnknownHostException: alc-vbox-testarchiveservice.minio archive-api | at java.base/java.net.InetAddress$CachedAddresses.get(InetAddress.java:797) archive-api | at java.base/java.net.InetAddress.getAllByName0(InetAddress.java:1519) archive-api | at java.base/java.net.InetAddress.getAllByName(InetAddress.java:1378) archive-api | at java.base/java.net.InetAddress.getAllByName(InetAddress.java:1306) archive-api | at com.amazonaws.SystemDefaultDnsResolver.resolve(SystemDefaultDnsResolver.java:27) archive-api | at com.amazonaws.http.DelegatingDnsResolver.resolve(DelegatingDnsResolver.java:38) archive-api | at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:112) archive-api | at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:376) archive-api | at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) archive-api | at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) archive-api | at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) archive-api | at java.base/java.lang.reflect.Method.invoke(Method.java:566) archive-api | at com.amazonaws.http.conn.ClientConnectionManagerFactory$Handler.invoke(ClientConnectionManagerFactory.java:76) archive-api | at com.amazonaws.http.conn.$Proxy97.connect(Unknown Source) archive-api | at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393) archive-api | at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) archive-api | at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) archive-api | at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) archive-api | at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) archive-api | at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) archive-api | at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72) archive-api | at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1346) archive-api | at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1157) archive-api | ... 77 more Does anyone know Why I am getting UnknownHostException? My doubt is since, I configured Minio-url(http://minio:9001) in the code where I am creating S3 configs, then why my code is trying to connect to alc-vbox-testarchiveservice.minio host? (My bucket name is: alc-vbox-testarchiveservice)
docker rabbit mq image fails to load up
i have a project that i am using rabbitmq image docker when i use this command in my project : docker-compose -f dc.rabbitmq.yml up i get the below error for failing to start the container : 477a2217c16b: Pull complete Digest: sha256:d8fb3795026b4c81eae33f4990e8bbc7b29c877388eef6ead2aca2945074c3f3 Status: Downloaded newer image for rabbitmq:3-management-alpine Creating rabbitmq ... done Attaching to rabbitmq rabbitmq | Configuring logger redirection rabbitmq | 07:50:26.108 [error] rabbitmq | rabbitmq | BOOT FAILED rabbitmq | 07:50:26.116 [error] BOOT FAILED rabbitmq | 07:50:26.117 [error] =========== rabbitmq | =========== rabbitmq | 07:50:26.117 [error] Exception during startup: rabbitmq | 07:50:26.117 [error] rabbitmq | Exception during startup: rabbitmq | rabbitmq | supervisor:'-start_children/2-fun-0-'/3 line 355 rabbitmq | 07:50:26.117 [error] supervisor:'-start_children/2-fun-0-'/3 line 355 rabbitmq | 07:50:26.117 [error] supervisor:do_start_child/2 line 371 rabbitmq | 07:50:26.117 [error] supervisor:do_start_child_i/3 line 385 rabbitmq | supervisor:do_start_child/2 line 371 rabbitmq | supervisor:do_start_child_i/3 line 385 rabbitmq | rabbit_prelaunch:run_prelaunch_first_phase/0 line 27 rabbitmq | rabbit_prelaunch:do_run/0 line 108 rabbitmq | 07:50:26.117 [error] rabbit_prelaunch:run_prelaunch_first_phase/0 line 27 rabbitmq | 07:50:26.117 [error] rabbit_prelaunch:do_run/0 line 108 rabbitmq | 07:50:26.117 [error] rabbit_prelaunch_conf:setup/1 line 33 rabbitmq | 07:50:26.117 [error] rabbit_prelaunch_conf:decrypt_config/2 line 404 rabbitmq | rabbit_prelaunch_conf:setup/1 line 33 rabbitmq | rabbit_prelaunch_conf:decrypt_config/2 line 404 rabbitmq | rabbit_prelaunch_conf:decrypt_app/3 line 425 rabbitmq | 07:50:26.117 [error] rabbit_prelaunch_conf:decrypt_app/3 line 425 rabbitmq | 07:50:26.117 [error] throw:{config_decryption_error,{key,default_pass},badarg} rabbitmq | throw:{config_decryption_error,{key,default_pass},badarg} rabbitmq | rabbitmq | 07:50:26.117 [error] rabbitmq | 07:50:27.118 [error] Supervisor rabbit_prelaunch_sup had child prelaunch started with rabbit_prelaunch:run_prelaunch_first_phase() at undefined exit with reason {config_decryption_error,{key,default_pass},badarg} in context start_error so what i have tried so far what trying to search for any child proccess who fails the rabbit but could not find any and removed images and container and build and up them again but again the same error . #EDIT here is the config file : version: "3.7" services: rabbitmq: image: rabbitmq:3-management-alpine container_name: rabbitmq ports: - "15672:15672" - "5672:5672" volumes: - ./docker/rabbitmq/definitions.json:/opt/definitions.json:ro - ./docker/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.config:ro - dms_rabbitmq_data:/var/lib/rabbitmq/ - dms_rabbitmq_logs:/var/log/rabbitmq networks: - rabbitmq_network labels: - co.elastic.logs/module=rabbitmq - co.elastic.logs/fileset.stdout=access - co.elastic.logs/fileset.stderr=error - co.elastic.metrics/module=rabbitmq - co.elastic.metrics/metricsets=status volumes: dms_rabbitmq_etc: name: dms_rabbitmq_etc dms_rabbitmq_data: name: dms_rabbitmq_data dms_rabbitmq_logs: driver: local driver_opts: type: "none" o: "bind" device: ${PWD}/storage/logs/rabbitmq networks: rabbitmq_network: name: rabbitmq_network
I had the same problem. The solution for me was recreating an encoded password.
error restarting JBoss/Keycloak docker container
I am setting up a keycloak server using the Jboss docker image(v.4.0.0.final) and I am encountering this error upon reboot: ..... keycloak_1 | at org.jboss.as.controller.AbstractControllerService$1.run(AbstractControllerService.java:370) keycloak_1 | at java.lang.Thread.run(Thread.java:748) keycloak_1 | keycloak_1 | 18:51:15,509 ERROR [org.jboss.as.controller.client] (Controller Boot Thread) WFLYCTL0190: Step handler org.jboss.as.server.DeployerChainAddHandler$FinalRuntimeStepHandler#52734583 for operation add-deployer-chains at address [] failed handling operation rollback -- java.util.concurrent.RejectedExecutionException: Task org.jboss.as.controller.notification.NotificationSupports$NonBlockingNotificationSupport$1#2211e00e rejected from java.util.concurrent.ThreadPoolExecutor#41133ff1[Shutting down, pool size = 50, active threads = 43, queued tasks = 0, completed tasks = 93] keycloak_1 | docker_keycloak_1 exited with code 1 When I run the docker container for the first time, everything goes as planned. I can login, create roles, etc. I launch the container using jhipster's keycloak script: version: '2' services: keycloak: image: jboss/keycloak:4.0.0.Final command: ["-b", "0.0.0.0", "-Dkeycloak.migration.action=import", "-Dkeycloak.migration.provider=dir", "-Dkeycloak.migration.dir=/opt/jboss/keycloak/realm-config", "-Dkeycloak.migration.strategy=OVERWRITE_EXISTING", "-Djboss.socket.binding.port-offset=1000"] volumes: - ./realm-config:/opt/jboss/keycloak/realm-config environment: - KEYCLOAK_USER=admin - KEYCLOAK_PASSWORD=admin - DB_DATABASE=postgres ports: - 9080:9080 - 9443:9443 - 10990:10990 Am I shutting it down correctly?
Docker compose - Cannot execute request on any known server
I have the following issue. I have two spring boot micro services (1.5.8 ver), a configuration service and a discovery service. When I run them locally the services are starting and communicating successfully. Now when I start this two services using a docker compose file I get the following error: Request execution failed with message: java.net.ConnectException: Connection refused (Connection refused) discovery-svc | WARN 24 --- [nfoReplicator-0] com.netflix.discovery.DiscoveryClient : DiscoveryClient_DISCOVERY/6c4a9f118423:discovery - registration failed Cannot execute request on any known server discovery-svc | discovery-svc | com.netflix.discovery.shared.transport.TransportException: Cannot execute request on any known server discovery-svc | at com.netflix.discovery.shared.transport.decorator.RetryableEurekaHttpClient.execute(RetryableEurekaHttpClient.java:111) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator$1.execute(EurekaHttpClientDecorator.java:59) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.shared.transport.decorator.SessionedEurekaHttpClient.execute(SessionedEurekaHttpClient.java:77) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.DiscoveryClient.register(DiscoveryClient.java:807) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.InstanceInfoReplicator.run(InstanceInfoReplicator.java:104) [eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_151] discovery-svc | at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_151] discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_151] discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_151] discovery-svc | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_151] discovery-svc | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_151] discovery-svc | at java.lang.Thread.run(Thread.java:748) [na:1.8.0_151] discovery-svc | discovery-svc | 2017-12-20 15:19:00.314 WARN 24 --- [nfoReplicator-0] c.n.discovery.InstanceInfoReplicator : There was a problem with the instance info replicator discovery-svc | discovery-svc | com.netflix.discovery.shared.transport.TransportException: Cannot execute request on any known server discovery-svc | at com.netflix.discovery.shared.transport.decorator.RetryableEurekaHttpClient.execute(RetryableEurekaHttpClient.java:111) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator$1.execute(EurekaHttpClientDecorator.java:59) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.shared.transport.decorator.SessionedEurekaHttpClient.execute(SessionedEurekaHttpClient.java:77) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator.register(EurekaHttpClientDecorator.java:56) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.DiscoveryClient.register(DiscoveryClient.java:807) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at com.netflix.discovery.InstanceInfoReplicator.run(InstanceInfoReplicator.java:104) ~[eureka-client-1.7.0.jar!/:1.7.0] discovery-svc | at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_151] discovery-svc | at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_151] discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_151] discovery-svc | at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_151] discovery-svc | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_151] discovery-svc | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_151] discovery-svc | at java.lang.Thread.run(Thread.java:748) [na:1.8.0_151] I have the following files: docker-compose.yml version: '3.4' services: config-svc: image: "config-img" container_name: config-svc build: context: ../config dockerfile: docker/config.dockerfile expose: - "8888" discovery-svc: image: "discovery-img" container_name: discovery-svc build: context: ../discovery dockerfile: docker/discovery.dockerfile depends_on: - config-svc links: - config-svc:config-svc entrypoint: sh -c '/tmp/wait-for config-svc:8888 && java -jar discovery.jar' expose: - "8761" Discovery service: bootstrap.yml spring: application: name: discovery cloud: config: uri: http://config-svc:8888 profiles: active: docker management: security: enabled: false Configuration service: bootstrap.yml server: port: 8888 spring: application: name: config cloud: config: server: native: search-locations: ${user.dir}/src/main/resources/configs profiles: active: - native security: basic: enabled: false management: security: enabled: false discovery.docker.properties server.port=8761 eureka.client.register-with-eureka=false eureka.client.fetch-registry=false eureka.client.serviceUrl.defaultZone=http://discovery-svc:8761/eureka/ eureka.instance.hostname=discovery-svc eureka.instance.preferIpAddress=false logging.level.com.netflix.eureka=OFF logging.level.com.netflix.discovery=OFF I have tried many things but with no luck. Can anyone suggest anything? Thanks in advance
are you has been try set network each service ? so, service config-svc and service discovery-svc communication through same network. try setup driver to bridge as follow version: '3.4' services: config-svc: image: "config-img" container_name: config-svc build: context: ../config dockerfile: docker/config.dockerfile expose: - "8888" networks: - app-network discovery-svc: image: "discovery-img" container_name: discovery-svc build: context: ../discovery dockerfile: docker/discovery.dockerfile depends_on: - config-svc links: - config-svc:config-svc entrypoint: sh -c '/tmp/wait-for config-svc:8888 && java -jar discovery.jar' expose: - "8761" networks: - app-network networks: app-network: driver: brige reference: https://docs.docker.com/compose/compose-file/
Ok I found it. The problem was in the configuration service in the bootstrap.yml file the property value of spring.cloud.config.server.native.search-locations must be fixed to classpath:configs. Now it's working fine. Thanks for your reply
unable to run kibana and logstash with elasticsearch
Elastic serach is running fine on 9201 port. But unable to run kibana and logstash with docker-compose. For logstash it throws the error: Attempted to resurrect connection to dead ES instance, but got an error. For kibana it throw warnings: "warning","elasticsearch","admin"],"pid":1,"message":"No living connections" Below is the docker-compose.yml file: version: '2' services: # Service 1 : elasticsearch elasticsearch-5-6: image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3 container_name: elasticsearch-5-6 ports: - "9201:9200" volumes: - /etc/elasticsearch/elasticsearch-5-6.yml:/usr/share/elasticsearch/config/elasticsearch.yml - /var/elasticsearch/data/immunedata-5-6/:/usr/share/elasticsearch/data/ #- /etc/elasticsearch/logging.yml:/usr/share/elasticsearch/config/logging.yml #- /var/log/elasticsearch/:/usr/share/elasticsearch/logs/ environment: - cluster.name=docker-cluster-elasticsearch-5-6 #- bootstrap.memory_lock=true - "ES_JAVA_OPTS: -Xmx2048m -Xms2048m" # Disabling the xpack security as it costs after one month of free trail. - xpack.security.enabled=false # Service 2 : logstash logstash-5-6: image: docker.elastic.co/logstash/logstash:5.6.3 container_name: logstash-5-6 ports: #- "5044:5044" - "5001:5001" volumes: - /etc/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml - /etc/logstash/pipeline:/usr/share/logstash/pipeline #- /etc/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml #- /var/logstash/pipeline:/usr/share/logstash/pipeline environment: - "ES_JAVA_OPTS: -Xmx2048m -Xms2048m" depends_on: - elasticsearch-5-6 # Service 3 : kibana kibana-5-6: image: docker.elastic.co/kibana/kibana:5.6.3 container_name: kibana-5-6 ports: - "5601:5601" volumes: - /etc/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml #- /var/kibana/immunedata-5-6/:/usr/share/kibana/data/ environment: - xpack.security.enabled=false - xpack.graph.enabled = false - xpack.ml.enabled = false - xpack.monitoring.enabled = false - xpack.watcher.enabled = false - xpack.reporting.enabled = false depends_on: - elasticsearch-5-6 # Service 4 : elasticseach-head elasticsearch-head: image: mobz/elasticsearch-head:5 container_name: elasticsearch-head # will not wait for elasticsearch to be ready. ports: - "9100:9100" elasticserach.yml cluster.name: immunedata-cluster-5.6 node.name: "immunedata-cluster-5-6.node-1" # Elasticsearch in docker access different data directory, defined mapping directory in docker-compose.yml #path.data: /var/elasticsearch/data/immunedata-5-6/ path.data: /usr/share/elasticsearch/data/ #path.data: /var/elasticsearch/data # NOTE : Since elasticsearch 5.x index level settings can NOT be set on the nodes configuration like the elasticsearch.yaml #index.number_of_shards: 1 #index.number_of_replicas: 0 # Allow all host access network.bind_host: 0.0.0.0 http.port: 9200 # To enable cross-origin resource sharing (Accessing on browser) http.cors.enabled: true http.cors.allow-origin : "*" logstash.yml file http.host: "0.0.0.0" path.config: /usr/share/logstash/pipeline #xpack.monitoring.elasticsearch.url: http://localhost:9201 ##xpack.monitoring.elasticsearch.url: http://elasticsearch:9201 #xpack.monitoring.elasticsearch.username: logstash_system #xpack.monitoring.elasticsearch.password: changeme xpack.monitoring.enabled: false kibana.yml file server.name: kibana server.host: "0" elasticsearch.url: http://192.168.56.10:9201 xpack.monitoring.ui.container.elasticsearch.enabled: false #elasticsearch.url: http://elasticsearch:9201 xpack.security.enabled: false ## Above I tired this - not working #elasticsearch.username: elastic #elasticsearch.password: changeme #xpack.monitoring.ui.container.elasticsearch.enabled: false #xpack.monitoring.ui.container.elasticsearch.enabled: true # Extra: ssl.verificationMode: false Logs: [elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_141] elasticsearch-5-6 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_141] elasticsearch-5-6 | at java.lang.Thread.run(Thread.java:748) [?:1.8.0_141] elasticsearch-5-6 | [2017-11-26T06:07:57,084][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][young][14][6] duration [18.2s], collections [1]/[18.5s], total [18.2s]/[23.5s], memory [178.2mb]->[79.5mb]/[1.9gb], all_pools {[young] [132.1mb]->[964kb]/[133.1mb]}{[survivor] [16.6mb]->[12.5mb]/[16.6mb]}{[old] [29.4mb]->[66.5mb]/[1.8gb]} elasticsearch-5-6 | [2017-11-26T06:07:57,085][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][14] overhead, spent [18.2s] collecting in the last [18.5s] elasticsearch-5-6 | [2017-11-26T06:07:57,298][ERROR][o.e.x.m.c.i.IndexRecoveryCollector] [immunedata-cluster-5-6.node-1] collector [index-recovery] failed to collect data elasticsearch-5-6 | org.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized]; elasticsearch-5-6 | at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:165) ~[elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction.checkGlobalBlock(TransportRecoveryAction.java:114) ~[elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction.checkGlobalBlock(TransportRecoveryAction.java:52) ~[elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.<init>(TransportBroadcastByNodeAction.java:256) ~[elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction.doExecute(TransportBroadcastByNodeAction.java:234) ~[elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction.doExecute(TransportBroadcastByNodeAction.java:79) ~[elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:170) ~[elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:142) ~[elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:84) ~[elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) ~[elasticsearch-5.6.3.jar:5.6.3] elasticsearch-5-6 | at elasticsearch-5-6 | [2017-11-26T06:08:45,238][WARN ][o.e.x.w.e.ExecutionService] [immunedata-cluster-5-6.node-1] Failed to execute watch [XYNCje-TQzKm9OLdiH60gQ_elasticsearch_cluster_status_60e3c208-acca-4462-ba47-0711279d8f5e-2017-11-26T06:08:35.573Z] elasticsearch-5-6 | [2017-11-26T06:08:54,886][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][young][63][9] duration [3.6s], collections [1]/[4.6s], total [3.6s]/[30.2s], memory [226.9mb]->[103.5mb]/[1.9gb], all_pools {[young] [127.5mb]->[1mb]/[133.1mb]}{[survivor] [16.6mb]->[11.3mb]/[16.6mb]}{[old] [82.7mb]->[91.2mb]/[1.8gb]} elasticsearch-5-6 | [2017-11-26T06:08:54,886][WARN ][o.e.m.j.JvmGcMonitorService] [immunedata-cluster-5-6.node-1] [gc][63] overhead, spent [3.6s] collecting in the last [4.6s] logstash-5-6 | Sending Logstash's logs to /usr/share/logstash/logs which is now configured via log4j2.properties elasticsearch-5-6 | [2017-11-26T06:08:55,988][INFO ][o.e.c.r.a.AllocationService] [immunedata-cluster-5-6.node-1] Cluster health status changed from [RED] to [YELLOW] (reason: [shards started [[.watcher-history-6-2017.11.20][0], [.monitoring-es-6-2017.11.20][0]] ...]). logstash-5-6 | [2017-11-26T06:08:56,786][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"} logstash-5-6 | [2017-11-26T06:08:56,891][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"} logstash-5-6 | [2017-11-26T06:08:57,558][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"arcsight", :directory=>"/usr/share/logstash/vendor/bundle/jruby/1.9/gems/x-pack-5.6.3-java/modules/arcsight/configuration"} logstash-5-6 | [2017-11-26T06:09:04,121][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elastic:xxxxxx#elasticsearch-5-6:9201/]}} logstash-5-6 | [2017-11-26T06:09:04,123][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"} elasticsearch-5-6 | [2017-11-26T06:09:04,687][WARN ][o.e.c.r.a.DiskThresholdMonitor] [immunedata-cluster-5-6.node-1] high disk watermark [90%] exceeded on [eAlcHBJ2QVG58e0HJsgrdQ][immunedata-cluster-5-6.node-1][/usr/share/elasticsearch/data/nodes/0] free: 1.9gb[7.4%], shards will be relocated away from this node elasticsearch-5-6 | [2017-11-26T06:09:04,687][INFO ][o.e.c.r.a.DiskThresholdMonitor] [immunedata-cluster-5-6.node-1] rerouting shards: [high disk watermark exceeded on one or more nodes] logstash-5-6 | [2017-11-26T06:09:06,450][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"} logstash-5-6 | [2017-11-26T06:09:06,452][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil} logstash-5-6 | [2017-11-26T06:09:06,455][ERROR][logstash.outputs.elasticsearch] Failed to install template. {:message=>"Template file '' could not be found!", :class=>"ArgumentError", :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/template_manager.rb:37:in `read_template_file'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/template_manager.rb:23:in `get_template'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/template_manager.rb:7:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/common.rb:58:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.2-java/lib/logstash/outputs/elasticsearch/common.rb:25:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:9:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/output_delegator.rb:43:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:290:in `register_plugin'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:310:in `start_workers'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:235:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:398:in `start_pipeline'"]} logstash-5-6 | [2017-11-26T06:09:06,455][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//elasticsearch-5-6:9201"]} logstash-5-6 | [2017-11-26T06:09:06,462][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>250} logstash-5-6 | [2017-11-26T06:09:09,818][INFO ][logstash.pipeline ] Pipeline main started logstash-5-6 | [2017-11-26T06:09:10,341][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600} logstash-5-6 | [2017-11-26T06:09:11,460][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"} logstash-5-6 | [2017-11-26T06:09:11,484][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"} logstash-5-6 | [2017-11-26T06:09:16,491][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"} logstash-5-6 | [2017-11-26T06:09:16,500][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:21Z","tags":["warning","elasticsearch","config","deprecation"],"pid":1,"message":"Config key \"ssl.verify\" is deprecated. It has been replaced with \"ssl.verificationMode\""} logstash-5-6 | [2017-11-26T06:09:21,513][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"} logstash-5-6 | [2017-11-26T06:09:21,523][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:26Z","tags":["status","plugin:kibana#5.6.3","info"],"pid":1,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"} logstash-5-6 | [2017-11-26T06:09:26,536][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"} logstash-5-6 | [2017-11-26T06:09:26,570][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:26Z","tags":["status","plugin:elasticsearch#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:26Z","tags":["status","plugin:xpack_main#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["status","plugin:graph#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["error","elasticsearch","admin"],"pid":1,"message":"Request error, retrying\nHEAD http://elasticsearch-5-6:9201/ => connect ECONNREFUSED 172.21.0.2:9201"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["status","plugin:monitoring#5.6.3","info"],"pid":1,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:29Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"} logstash-5-6 | [2017-11-26T06:09:31,585][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"} logstash-5-6 | [2017-11-26T06:09:31,603][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["reporting","warning"],"pid":1,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent pending reports from failing on restart, please set xpack.reporting.encryptionKey in kibana.yml"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:reporting#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:xpack_main#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:graph#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:reporting#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:elasticsearch#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from yellow to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:33Z","tags":["status","plugin:searchprofiler#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from uninitialized to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"uninitialized","prevMsg":"uninitialized"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:34Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:34Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:34Z","tags":["status","plugin:ml#5.6.3","error"],"pid":1,"state":"red","message":"Status changed from uninitialized to red - Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201.","prevState":"uninitialized","prevMsg":"uninitialized"} elasticsearch-5-6 | [2017-11-26T06:09:34,750][WARN ][o.e.c.r.a.DiskThresholdMonitor] [immunedata-cluster-5-6.node-1] high disk watermark [90%] exceeded on [eAlcHBJ2QVG58e0HJsgrdQ][immunedata-cluster-5-6.node-1][/usr/share/elasticsearch/data/nodes/0] free: 1.9gb[7.4%], shards will be relocated away from this node logstash-5-6 | [2017-11-26T06:09:36,692][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://elastic:xxxxxx#elasticsearch-5-6:9201/, :path=>"/"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["status","plugin:ml#5.6.3","info"],"pid":1,"state":"yellow","message":"Status changed from red to yellow - Waiting for Elasticsearch","prevState":"red","prevMsg":"Unable to connect to Elasticsearch at http://elasticsearch-5-6:9201."} logstash-5-6 | [2017-11-26T06:09:37,366][WARN ][logstash.outputs.elasticsearch] Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx#elasticsearch-5-6:9201/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx#elasticsearch-5-6:9201/][Manticore::SocketException] Connection refused (Connection refused)"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://elasticsearch-5-6:9201/"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"No living connections"} kibana-5-6 | {"type":"log","#timestamp":"2017-11-26T06:09:37Z","tags":
You called elasticsearch service elasticsearch-5-6 in your docker-compose.yml. That means that container with elasticsearch is available on address http://elasticsearch-5-6:9200 for all other containers in your docker-compose.yaml. And it is available on address http://127.0.0.1:9201 from the host machine. In order to have workable ELK stack you need to change logstash config to: http.host: "0.0.0.0" path.config: /usr/share/logstash/pipeline xpack.monitoring.elasticsearch.url: http://elasticsearch-5-6:9200 #xpack.monitoring.elasticsearch.username: logstash_system #xpack.monitoring.elasticsearch.password: changeme xpack.monitoring.enabled: false and kibana config to: server.name: kibana server.host: "0" elasticsearch.url: http://elasticsearch-5-6:9200 xpack.monitoring.ui.container.elasticsearch.enabled: false xpack.security.enabled: false ## Above I tired this - not working #elasticsearch.username: elastic #elasticsearch.password: changeme #xpack.monitoring.ui.container.elasticsearch.enabled: false #xpack.monitoring.ui.container.elasticsearch.enabled: true # Extra: ssl.verificationMode: false
EKL Cluster with Xpack disabled You are missing with the ELASTICSEARCH_URL: "http://elasticsearch:9200" in kibana and xpack.monitoring.elasticsearch.url: http://elasticsearch:9200 in Logstash here is the sample yml configuration ith all possible environment varibales defined in environment version: '3.4' services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0 container_name: elasticsearch environment: ES_JAVA_OPTS: '-Xms2048m -Xmx2048m' cluster.name: es-cluster node.name: es1 network.bind_host: 0.0.0.0 discovery.zen.minimum_master_nodes: 1 discovery.zen.ping.unicast.hosts: elasticsearch1 xpack.security.enabled: 'false' xpack.monitoring.enabled: 'false' xpack.watcher.enabled: 'false' xpack.ml.enabled: 'false' http.cors.enabled : 'true' http.cors.allow-origin : "*" http.cors.allow-methods : OPTIONS, HEAD, GET, POST, PUT, DELETE http.cors.allow-headers : X-Requested-With,X-Auth-Token,Content-Type, Content-Length logger.level: debug volumes: - /var/elasticsearch/db/elasticsearch/data:/usr/share/elasticsearch/data ports: - 9200:9200 - 9300:9300 networks: - elastic logstash: image: docker.elastic.co/logstash/logstash:6.6.0 container_name: logstash ports: - 5044:5044 - 5001:5001 volumes: - /var/elasticsearch/logstash/pipeline:/usr/share/logstash/pipeline environment: ES_JAVA_OPTS: -Xmx2048m -Xms2048m" http.host: 0.0.0.0 xpack.monitoring.enabled: 'false' xpack.monitoring.elasticsearch.url: http://elasticsearch:9200 networks: - elastic depends_on: - elasticsearch kibana: image: docker.elastic.co/kibana/kibana:6.6.0 container_name: kibana environment: ELASTICSEARCH_URL: "http://elasticsearch:9200" xpack.security.enabled: 'false' xpack.graph.enabled : 'false' xpack.ml.enabled : 'false' xpack.monitoring.enabled : 'false' xpack.watcher.enabled : 'false' xpack.reporting.enabled : 'false' ports: - 5601:5601 networks: - elastic depends_on: - elasticsearch elasticsearch-head: image: mobz/elasticsearch-head:5 container_name: elasticsearch-head ports: - "9100:9100" networks: - elastic networks: elastic: driver: bridge