I am trying to deploy my spark application on Kubernetes. I followed the below steps:
Installed spark-kubernetes-operator:
helm repo add spark-operator https://googlecloudplatform.github.io/spark-on-k8s-operator
helm install gcp-spark-operator spark-operator/spark-operator
Created a spark-app.py
from pyspark.sql.functions import *
from pyspark.sql import *
from pyspark import SparkContext
from pyspark.sql import SparkSession
from pyspark.sql.types import StructType,StructField, StringType, IntegerType
if __name__ == "__main__":
spark = SparkSession.builder.appName('spark-on-kubernetes-test').getOrCreate()
data2 = [("James","","Smith","36636","M",3000),
("Michael","Rose","","40288","M",4000),
("Robert","","Williams","42114","M",4000),
("Maria","Anne","Jones","39192","F",4000),
("Jen","Mary","Brown","","F",-1)
]
schema = StructType([ \
StructField("firstname",StringType(),True), \
StructField("middlename",StringType(),True), \
StructField("lastname",StringType(),True), \
StructField("id", StringType(), True), \
StructField("gender", StringType(), True), \
StructField("salary", IntegerType(), True) \
])
df = spark.createDataFrame(data=data2,schema=schema)
df.printSchema()
df.show(truncate=False)
print("program is completed !")
Then I created the new image with my application:
FROM bitnami/spark
USER root
WORKDIR /app
COPY requirements.txt .
RUN pip3 install -r requirements.txt
COPY spark-app.py .<
Then I created the spark-application.yaml file:
apiVersion: "sparkoperator.k8s.io/v1beta2"
kind: SparkApplication
metadata:
name: pyspark-app
namespace: default
spec:
type: Python
mode: cluster
image: "test/spark-k8s-app:1.0"
imagePullPolicy: Always
mainApplicationFile: local:///app/spark-app.py
sparkVersion: 3.3.0
restartPolicy:
type: OnFailure
onFailureRetries: 3
onFailureRetryInterval: 10
onSubmissionFailureRetries: 5
onSubmissionFailureRetryInterval: 20
driver:
cores: 1
coreLimit: "1200m"
memory: "512m"
serviceAccount: spark
labels:
version: 3.3.0
volumeMounts:
- name: "test-volume"
mountPath: "/tmp"
executor:
cores: 1
instances: 1
memory: "512m"
labels:
version: 3.3.0
volumeMounts:
- name: "test-volume"
mountPath: "/tmp"
But when I tried to deploy the yaml file, I am getting the below error:
$ kubectl logs pyspark-app-driver
←[38;5;6m ←[38;5;5m13:04:01.12 ←[0m
←[38;5;6m ←[38;5;5m13:04:01.12 ←[0m←[1mWelcome to the Bitnami spark container←[0m
←[38;5;6m ←[38;5;5m13:04:01.12 ←[0mSubscribe to project updates by watching ←[1mhttps://github.com/bitnami/containers←[0m
←[38;5;6m ←[38;5;5m13:04:01.12 ←[0mSubmit issues and feature requests at ←[1mhttps://github.com/bitnami/containers/issues←[0m
←[38;5;6m ←[38;5;5m13:04:01.12 ←[0m
22/09/24 13:04:04 INFO SparkContext: Running Spark version 3.3.0
22/09/24 13:04:04 INFO ResourceUtils: ==============================================================
22/09/24 13:04:04 INFO ResourceUtils: No custom resources configured for spark.driver.
22/09/24 13:04:04 INFO ResourceUtils: ==============================================================
22/09/24 13:04:04 INFO SparkContext: Submitted application: ml_framework
22/09/24 13:04:04 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 512, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)
22/09/24 13:04:04 INFO ResourceProfile: Limiting resource is cpus at 1 tasks per executor
22/09/24 13:04:04 INFO ResourceProfileManager: Added ResourceProfile id: 0
22/09/24 13:04:04 INFO SecurityManager: Changing view acls to: root
22/09/24 13:04:04 INFO SecurityManager: Changing modify acls to: root
22/09/24 13:04:04 INFO SecurityManager: Changing view acls groups to:
22/09/24 13:04:04 INFO SecurityManager: Changing modify acls groups to:
22/09/24 13:04:04 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users
with view permissions: Set(root); groups with view permissions: Set(); users with modify permissions: Set(root); groups with modify permissions: Set()
22/09/24 13:04:05 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
22/09/24 13:04:05 INFO Utils: Successfully started service 'sparkDriver' on port 7078.
22/09/24 13:04:05 INFO SparkEnv: Registering MapOutputTracker
22/09/24 13:04:05 INFO SparkEnv: Registering BlockManagerMaster
22/09/24 13:04:05 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
22/09/24 13:04:05 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up
22/09/24 13:04:05 INFO SparkEnv: Registering BlockManagerMasterHeartbeat
22/09/24 13:04:05 INFO DiskBlockManager: Created local directory at /var/data/spark-019ba05b-dba8-4350-a281-ffa35b54d840/blockmgr-20d5c478-8e93-42f6-85a9-9ed070f50b2b
22/09/24 13:04:05 INFO MemoryStore: MemoryStore started with capacity 117.0 MiB
22/09/24 13:04:05 INFO SparkEnv: Registering OutputCommitCoordinator
22/09/24 13:04:06 INFO Utils: Successfully started service 'SparkUI' on port 4040.
22/09/24 13:04:06 INFO StandaloneAppClient$ClientEndpoint: Connecting to master spark://spark-master:7077...
22/09/24 13:04:10 WARN TransportClientFactory: DNS resolution failed for spark-master:7077 took 4005 ms
22/09/24 13:04:10 WARN StandaloneAppClient$ClientEndpoint: Failed to connect to master spark-master:7077
org.apache.spark.SparkException: Exception thrown in awaitResult:
at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:301)
at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102)
at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110)
at org.apache.spark.deploy.client.StandaloneAppClient$ClientEndpoint$$anon$1.run(StandaloneAppClient.scala:107)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.io.IOException: Failed to connect to spark-master:7077
at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:288)
at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:218)
at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:230)
at org.apache.spark.rpc.netty.NettyRpcEnv.createClient(NettyRpcEnv.scala:204)
at org.apache.spark.rpc.netty.Outbox$$anon$1.call(Outbox.scala:202)
at org.apache.spark.rpc.netty.Outbox$$anon$1.call(Outbox.scala:198)
... 4 more
Caused by: java.net.UnknownHostException: spark-master
at java.net.InetAddress.getAllByName0(InetAddress.java:1287)
at java.net.InetAddress.getAllByName(InetAddress.java:1199)
at java.net.InetAddress.getAllByName(InetAddress.java:1127)
at java.net.InetAddress.getByName(InetAddress.java:1077)
at io.netty.util.internal.SocketUtils$8.run(SocketUtils.java:156)
at io.netty.util.internal.SocketUtils$8.run(SocketUtils.java:153)
at java.security.AccessController.doPrivileged(Native Method)
at io.netty.util.internal.SocketUtils.addressByName(SocketUtils.java:153)
at io.netty.resolver.DefaultNameResolver.doResolve(DefaultNameResolver.java:41)
at io.netty.resolver.SimpleNameResolver.resolve(SimpleNameResolver.java:61)
at io.netty.resolver.SimpleNameResolver.resolve(SimpleNameResolver.java:53)
at io.netty.resolver.InetSocketAddressResolver.doResolve(InetSocketAddressResolver.java:55)
at io.netty.resolver.InetSocketAddressResolver.doResolve(InetSocketAddressResolver.java:31)
at io.netty.resolver.AbstractAddressResolver.resolve(AbstractAddressResolver.java:106)
at io.netty.bootstrap.Bootstrap.doResolveAndConnect0(Bootstrap.java:206)
at io.netty.bootstrap.Bootstrap.access$000(Bootstrap.java:46)
at io.netty.bootstrap.Bootstrap$1.operationComplete(Bootstrap.java:180)
at io.netty.bootstrap.Bootstrap$1.operationComplete(Bootstrap.java:166)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:552)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616)
at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605)
at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)
at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)
at io.netty.channel.AbstractChannel$AbstractUnsafe.safeSetSuccess(AbstractChannel.java:990)
at io.netty.channel.AbstractChannel$AbstractUnsafe.register0(AbstractChannel.java:516)
at io.netty.channel.AbstractChannel$AbstractUnsafe.access$200(AbstractChannel.java:429)
at io.netty.channel.AbstractChannel$AbstractUnsafe$1.run(AbstractChannel.java:486)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:503)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
... 1 more
22/09/24 13:04:26 INFO StandaloneAppClient$ClientEndpoint: Connecting to master spark://spark-master:7077...
22/09/24 13:04:30 WARN TransportClientFactory: DNS resolution failed for spark-master:7077 took 4006 ms
22/09/24 13:04:30 WARN StandaloneAppClient$ClientEndpoint: Failed to connect to master spark-master:7077
org.apache.spark.SparkException: Exception thrown in awaitResult:
at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:301)
at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102)
at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110)
at org.apache.spark.deploy.client.StandaloneAppClient$ClientEndpoint$$anon$1.run(StandaloneAppClient.scala:107)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.io.IOException: Failed to connect to spark-master:7077
at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:288)
at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:218)
at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:230)
at org.apache.spark.rpc.netty.NettyRpcEnv.createClient(NettyRpcEnv.scala:204)
at org.apache.spark.rpc.netty.Outbox$$anon$1.call(Outbox.scala:202)
at org.apache.spark.rpc.netty.Outbox$$anon$1.call(Outbox.scala:198)
... 4 more
Caused by: java.net.UnknownHostException: spark-master
at java.net.InetAddress.getAllByName0(InetAddress.java:1287)
at java.net.InetAddress.getAllByName(InetAddress.java:1199)
at java.net.InetAddress.getAllByName(InetAddress.java:1127)
at java.net.InetAddress.getByName(InetAddress.java:1077)
at io.netty.util.internal.SocketUtils$8.run(SocketUtils.java:156)
at io.netty.util.internal.SocketUtils$8.run(SocketUtils.java:153)
at java.security.AccessController.doPrivileged(Native Method)
at io.netty.util.internal.SocketUtils.addressByName(SocketUtils.java:153)
at io.netty.resolver.DefaultNameResolver.doResolve(DefaultNameResolver.java:41)
at io.netty.resolver.SimpleNameResolver.resolve(SimpleNameResolver.java:61)
at io.netty.resolver.SimpleNameResolver.resolve(SimpleNameResolver.java:53)
at io.netty.resolver.InetSocketAddressResolver.doResolve(InetSocketAddressResolver.java:55)
at io.netty.resolver.InetSocketAddressResolver.doResolve(InetSocketAddressResolver.java:31)
at io.netty.resolver.AbstractAddressResolver.resolve(AbstractAddressResolver.java:106)
at io.netty.bootstrap.Bootstrap.doResolveAndConnect0(Bootstrap.java:206)
at io.netty.bootstrap.Bootstrap.access$000(Bootstrap.java:46)
at io.netty.bootstrap.Bootstrap$1.operationComplete(Bootstrap.java:180)
at io.netty.bootstrap.Bootstrap$1.operationComplete(Bootstrap.java:166)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:552)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616)
at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605)
at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)
at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)
at io.netty.channel.AbstractChannel$AbstractUnsafe.safeSetSuccess(AbstractChannel.java:990)
at io.netty.channel.AbstractChannel$AbstractUnsafe.register0(AbstractChannel.java:516)
at io.netty.channel.AbstractChannel$AbstractUnsafe.access$200(AbstractChannel.java:429)
at io.netty.channel.AbstractChannel$AbstractUnsafe$1.run(AbstractChannel.java:486)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469)9)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:503)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
... 1 more
How can I resolve this?
Related
I’m attempting to execute a Jenkins & Docker CLI container on Kubernetes. Here are my steps:
I create the pod using:
kubectl --kubeconfig my-kubeconfig.yml run my-jenkins-pod --image=trion/jenkins-docker-client --restart=Never
Which creates a pod-based on the image https://hub.docker.com/r/trion/jenkins-docker-client
I create the deployment using:
kubectl --kubeconfig my-kubeconfig.yml apply -f /kub/kube
/kub/kube contains jenkins-deployment-yaml which I have configured as:
apiVersion: v1
kind: Service
metadata:
name: my-jenkins-pod
spec:
ports:
- protocol: "TCP"
port: 50000
targetPort: 5001
selector:
app: my-jenkins-pod
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: my-jenkins-pod
spec:
selector:
matchLabels:
app: my-jenkins-pod
replicas: 1
template:
metadata:
labels:
app: my-jenkins-pod
spec:
containers:
- name: ml-services
image: trion/jenkins-docker-client
ports:
- containerPort: 5001
To access the Jenkins container I expose the IP using:
kubectl --kubeconfig my-kubeconfig.yml expose deployment my-jenkins-pod --type=LoadBalancer --name=my-jenkins-pod-public
To return the IP of the Jenkins and Docker image I use :
kubectl --kubeconfig my-kubeconfig.yml get services my-jenkins-pod-public
Which returns:
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
my-jenkins-pod-public LoadBalancer 9.5.52.28 161.61.222.16 5001:30878/TCP 10m
To test I open the URL at location:
http://161.61.222.16:5001/
Which returns:
This page isn’t working
161.61.222.16 didn’t send any data.
ERR_EMPTY_RESPONSE
It seems the service has started but the port mappings are incorrect?
The log of the pod my-jenkins-pod contains:
Running from: /usr/share/jenkins/jenkins.war webroot:
EnvVars.masterEnvVars.get("JENKINS_HOME") 2021-04-03 11:15:42.899+0000
[id=1] INFO org.eclipse.jetty.util.log.Log#initialized: Logging
initialized #274ms to org.eclipse.jetty.util.log.JavaUtilLog
2021-04-03 11:15:43.012+0000 [id=1] INFO winstone.Logger#logInternal:
Beginning extraction from war file 2021-04-03 11:15:44.369+0000
[id=1] WARNING o.e.j.s.handler.ContextHandler#setContextPath: Empty
contextPath 2021-04-03 11:15:44.416+0000
[id=1] INFO org.eclipse.jetty.server.Server#doStart:
jetty-9.4.39.v20210325; built: 2021-03-25T14:42:11.471Z; git:
9fc7ca5a922f2a37b84ec9dbc26a5168cee7e667; jvm 1.8.0_282-b08 2021-04-03
11:15:44.653+0000
[id=1] INFO o.e.j.w.StandardDescriptorProcessor#visitServlet: NO JSP
Support for /, did not find org.eclipse.jetty.jsp.JettyJspServlet
2021-04-03 11:15:44.695+0000
[id=1] INFO o.e.j.s.s.DefaultSessionIdManager#doStart:
DefaultSessionIdManager workerName=node0 2021-04-03 11:15:44.695+0000
[id=1] INFO o.e.j.s.s.DefaultSessionIdManager#doStart: No
SessionScavenger set, using defaults 2021-04-03 11:15:44.696+0000
[id=1] INFO o.e.j.server.session.HouseKeeper#startScavenging: node0
Scavenging every 660000ms 2021-04-03 11:15:45.081+0000
[id=1] INFO hudson.WebAppMain#contextInitialized: Jenkins home
directory: /var/jenkins_home found at:
EnvVars.masterEnvVars.get("JENKINS_HOME") 2021-04-03 11:15:45.203+0000
[id=1] INFO o.e.j.s.handler.ContextHandler#doStart: Started
w.#24f43aa3{Jenkins
v2.286,/,file:///var/jenkins_home/war/,AVAILABLE}{/var/jenkins_home/war}
2021-04-03 11:15:45.241+0000
[id=1] INFO o.e.j.server.AbstractConnector#doStart: Started
ServerConnector#4f0f2942{HTTP/1.1, (http/1.1)}{0.0.0.0:8080}
2021-04-03 11:15:45.241+0000
[id=1] INFO org.eclipse.jetty.server.Server#doStart: Started #2616ms
2021-04-03 11:15:45.245+0000 [id=21] INFO winstone.Logger#logInternal:
Winstone Servlet Engine running: controlPort=disabled 2021-04-03
11:15:46.479+0000 [id=26] INFO jenkins.InitReactorRunner$1#onAttained:
Started initialization 2021-04-03 11:15:46.507+0000
[id=26] INFO jenkins.InitReactorRunner$1#onAttained: Listed all
plugins 2021-04-03 11:15:47.654+0000
[id=27] INFO jenkins.InitReactorRunner$1#onAttained: Prepared all
plugins 2021-04-03 11:15:47.660+0000
[id=26] INFO jenkins.InitReactorRunner$1#onAttained: Started all
plugins 2021-04-03 11:15:47.680+0000
[id=27] INFO jenkins.InitReactorRunner$1#onAttained: Augmented all
extensions 2021-04-03 11:15:48.620+0000
[id=26] INFO jenkins.InitReactorRunner$1#onAttained: System config
loaded 2021-04-03 11:15:48.621+0000
[id=26] INFO jenkins.InitReactorRunner$1#onAttained: System config
adapted 2021-04-03 11:15:48.621+0000
[id=27] INFO jenkins.InitReactorRunner$1#onAttained: Loaded all jobs
2021-04-03 11:15:48.622+0000
[id=27] INFO jenkins.InitReactorRunner$1#onAttained: Configuration for
all jobs updated 2021-04-03 11:15:48.704+0000
[id=40] INFO hudson.model.AsyncPeriodicWork#lambda$doRun$0: Started
Download metadata 2021-04-03 11:15:48.722+0000
[id=40] INFO hudson.util.Retrier#start: Attempt #1 to do the action
check updates server 2021-04-03 11:15:49.340+0000
[id=26] INFO jenkins.install.SetupWizard#init:
************************************************************* Jenkins initial setup is required. An admin user has been created and a
password generated. Please use the following password to proceed to
installation: ab5dbf74145c405fb5a33456d4b97436 This may also be found
at: /var/jenkins_home/secrets/initialAdminPassword
************************************************************* 2021-04-03 11:16:08.107+0000
[id=27] INFO jenkins.InitReactorRunner$1#onAttained: Completed
initialization 2021-04-03 11:16:08.115+0000
[id=20] INFO hudson.WebAppMain$3#run: Jenkins is fully up and running
2021-04-03 11:16:08.331+0000
[id=40] INFO h.m.DownloadService$Downloadable#load: Obtained the
updated data file for hudson.tasks.Maven.MavenInstaller 2021-04-03
11:16:08.332+0000 [id=40] INFO hudson.util.Retrier#start: Performed
the action check updates server successfully at the attempt #1
2021-04-03 11:16:08.334+0000
[id=40] INFO hudson.model.AsyncPeriodicWork#lambda$doRun$0: Finished
Download metadata. 19,626 ms
Is Jenkins server is started at port 8080? because of this log message:
11:15:45.241+0000 [id=1] INFO o.e.j.server.AbstractConnector#doStart:
Started ServerConnector#4f0f2942{HTTP/1.1, (http/1.1)}{0.0.0.0:8080}
2021-04-03
I tried changing jenkins-deployment-yaml to point at port 8080 instead of 50000, resulting in the updated jenkins-deployment-yaml :
apiVersion: v1
kind: Service
metadata:
name: my-jenkins-pod
spec:
ports:
- protocol: "TCP"
port: 8080
But the same error is returned when I attempt to access http://161.61.222.16:5001/
Are my port mappings incorrect? Is this a correct method of adding an existing docker container that is available on the docker hub to a Kubernetes cluster?
Update:
The result of command kubectl describe services my-jenkins-pod-public is :
Name: my-jenkins-pod-public
Namespace: default
Labels: <none>
Annotations: kubernetes.digitalocean.com/load-balancer-id: d46ae9ae-6e8a-4fd8-aa58-43c08310059a
Selector: app=my-jenkins-pod
Type: LoadBalancer
IP Families: <none>
IP: 10.245.152.228
IPs: 10.245.152.228
LoadBalancer Ingress: 161.61.222.16
Port: <unset> 5001/TCP
TargetPort: 5001/TCP
NodePort: <unset> 30878/TCP
Endpoints: 10.214.12.12:5001
Session Affinity: None
External Traffic Policy: Cluster
Events: <none>
Trying to access http://161.61.222.16:30878/ via browser returns:
This site can’t be reached159.65.211.46 refused to connect. Try:
Checking the connection Checking the proxy and the firewall
ERR_CONNECTION_REFUSED
Trying to access http://161.61.222.16:5001/ via browser returns:
This page isn’t working
161.61.222.16 didn’t send any data.
ERR_EMPTY_RESPONSE
Seems the port 5001 is exposed/accessible but is not sending any data.
I also tried accessing 10.214.12.12 on ports 5001 & 30878 but both requests time out.
You need to use http://161.61.222.16:30878/ from outside of the host which is running containers on. Port 5001 is just accessible inside the cluster with internal IP (9.5.52.28 is in your case). Whenever you expose your deployment, automatically (also you can define manually) one of the NodePort (by default between 30000 - 32767)assign to the service for external request.
For service details, you need to run the below command. The command output will give you NodePort and another details.
kubectl describe services my-service
Please check related kubernetes documentation
Also you have configured service with port 5001 but Jenkins working with 8080 as far as I see in logs. Try to change target port of service to 8080 from 5001
I have a spark cluster setup on docker. using a openjdk:8-alpine. I am running it on local. Local pc name is DESKTOP-PCH5L6D
I am able to submit jobs but the error I see at the worker docker container is
Caused by: java.net.UnknownHostException: DESKTOP-PCH5L6D
local pc is on Win 10 Pro.
Am new to docker and seeking help as to how to approach to solve for this.
docker compose file am using to create the local spark cluster
version: "3.7"
services:
spark-master:
image: cmishr4/spark:latest
container_name: spark-master
hostname: spark-master
ports:
- "8080:8080"
- "7077:7077"
volumes:
- ./../share:/share
privileged: true
networks:
- spark-network
environment:
- "SPARK_MASTER_PORT=7077"
- "SPARK_MASTER_WEBUI_PORT=8080"
command: "sh start-master.sh"
spark-worker:
image: cmishr4/spark:latest
depends_on:
- spark-master
ports:
- 8080
volumes:
- ./../share:/share
privileged: true
networks:
- spark-network
environment:
- "SPARK_MASTER=spark://spark-master:7077"
- "SPARK_WORKER_WEBUI_PORT=8080"
command: "sh start-worker.sh"
networks:
spark-network:
driver: bridge
ipam:
driver: default
scala class main method:
val spark = SparkSession.builder
.appName("sample")
.master("spark://localhost:7077")
.config("spark.executor.cores", "1")
.config("spark.executor.memory","1g")
.getOrCreate()
val csv = spark.sparkContext.textFile("C:/Users/chand/wrkspcs/sparkws/share/input")
val rows = csv.map(line => line.split(",").map(_.trim))
val header = rows.first
val data = rows.filter(_(0) != header(0))
val rdd = data.map(row => Row(row(0), row(1).toInt))
val schema = new StructType()
.add(StructField("name", StringType, true))
.add(StructField("age", IntegerType, true))
val df = spark.sqlContext.createDataFrame(rdd, schema)
spark.stop()
log from the main class
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
19/05/27 02:25:55 INFO SparkContext: Running Spark version 2.4.3
19/05/27 02:25:55 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
19/05/27 02:25:55 INFO SparkContext: Submitted application: sample
19/05/27 02:25:56 INFO SecurityManager: Changing view acls to: chand
19/05/27 02:25:56 INFO SecurityManager: Changing modify acls to: chand
19/05/27 02:25:56 INFO SecurityManager: Changing view acls groups to:
19/05/27 02:25:56 INFO SecurityManager: Changing modify acls groups to:
19/05/27 02:25:56 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(chand); groups with view permissions: Set(); users with modify permissions: Set(chand); groups with modify permissions: Set()
<b>19/05/27 02:25:57 INFO Utils: Successfully started service 'sparkDriver' on port 53511.</b>
19/05/27 02:25:57 INFO SparkEnv: Registering MapOutputTracker
19/05/27 02:25:57 INFO SparkEnv: Registering BlockManagerMaster
19/05/27 02:25:57 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
19/05/27 02:25:57 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up
19/05/27 02:25:57 INFO DiskBlockManager: Created local directory at C:\Users\chand\AppData\Local\Temp\blockmgr-c4f6d60f-c867-4c52-895d-111030c92193
19/05/27 02:25:57 INFO MemoryStore: MemoryStore started with capacity 4.1 GB
19/05/27 02:25:57 INFO SparkEnv: Registering OutputCommitCoordinator
19/05/27 02:25:57 INFO Utils: Successfully started service 'SparkUI' on port 4040.
19/05/27 02:25:57 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://DESKTOP-PCH5L6D:4040
19/05/27 02:25:57 INFO StandaloneAppClient$ClientEndpoint: Connecting to master spark://localhost:7077...
19/05/27 02:25:57 INFO TransportClientFactory: Successfully created connection to localhost/127.0.0.1:7077 after 27 ms (0 ms spent in bootstraps)
19/05/27 02:25:57 INFO StandaloneSchedulerBackend: Connected to Spark cluster with app ID app-20190527062557-0000
19/05/27 02:25:57 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53553.
19/05/27 02:25:57 INFO NettyBlockTransferService: Server created on DESKTOP-PCH5L6D:53553
19/05/27 02:25:57 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
19/05/27 02:25:57 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, DESKTOP-PCH5L6D, 53553, None)
19/05/27 02:25:57 INFO StandaloneAppClient$ClientEndpoint: Executor added: app-20190527062557-0000/0 on worker-20190527062533-172.18.0.3-44351 (172.18.0.3:44351) with 1 core(s)
19/05/27 02:25:57 INFO StandaloneSchedulerBackend: Granted executor ID app-20190527062557-0000/0 on hostPort 172.18.0.3:44351 with 1 core(s), 1024.0 MB RAM
19/05/27 02:25:57 INFO BlockManagerMasterEndpoint: Registering block manager DESKTOP-PCH5L6D:53553 with 4.1 GB RAM, BlockManagerId(driver, DESKTOP-PCH5L6D, 53553, None)
19/05/27 02:25:57 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, DESKTOP-PCH5L6D, 53553, None)
19/05/27 02:25:57 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, DESKTOP-PCH5L6D, 53553, None)
19/05/27 02:25:57 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20190527062557-0000/0 is now RUNNING
19/05/27 02:25:57 INFO StandaloneSchedulerBackend: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.0
19/05/27 02:25:58 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 214.6 KB, free 4.1 GB)
19/05/27 02:25:58 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 20.4 KB, free 4.1 GB)
19/05/27 02:25:58 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on DESKTOP-PCH5L6D:53553 (size: 20.4 KB, free: 4.1 GB)
19/05/27 02:25:58 INFO SparkContext: Created broadcast 0 from textFile at App.scala:24
19/05/27 02:25:58 INFO FileInputFormat: Total input paths to process : 2
19/05/27 02:25:58 INFO SparkContext: Starting job: first at App.scala:26
19/05/27 02:25:58 INFO DAGScheduler: Got job 0 (first at App.scala:26) with 1 output partitions
19/05/27 02:25:58 INFO DAGScheduler: Final stage: ResultStage 0 (first at App.scala:26)
19/05/27 02:25:58 INFO DAGScheduler: Parents of final stage: List()
19/05/27 02:25:58 INFO DAGScheduler: Missing parents: List()
19/05/27 02:25:58 INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[2] at map at App.scala:25), which has no missing parents
19/05/27 02:25:58 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 4.1 GB)
19/05/27 02:25:58 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.2 KB, free 4.1 GB)
19/05/27 02:25:58 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on DESKTOP-PCH5L6D:53553 (size: 2.2 KB, free: 4.1 GB)
19/05/27 02:25:58 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1161
19/05/27 02:25:58 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at App.scala:25) (first 15 tasks are for partitions Vector(0))
19/05/27 02:25:58 INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks
19/05/27 02:25:59 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20190527062557-0000/0 is now EXITED (Command exited with code 1)
19/05/27 02:25:59 INFO StandaloneSchedulerBackend: Executor app-20190527062557-0000/0 removed: Command exited with code 1
19/05/27 02:25:59 INFO StandaloneAppClient$ClientEndpoint: Executor added: app-20190527062557-0000/1 on worker-20190527062533-172.18.0.3-44351 (172.18.0.3:44351) with 1 core(s)
19/05/27 02:25:59 INFO StandaloneSchedulerBackend: Granted executor ID app-20190527062557-0000/1 on hostPort 172.18.0.3:44351 with 1 core(s), 1024.0 MB RAM
19/05/27 02:25:59 INFO BlockManagerMaster: Removal of executor 0 requested
19/05/27 02:25:59 INFO CoarseGrainedSchedulerBackend$DriverEndpoint: Asked to remove non-existent executor 0
19/05/27 02:25:59 INFO BlockManagerMasterEndpoint: Trying to remove executor 0 from BlockManagerMaster.
19/05/27 02:25:59 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20190527062557-0000/1 is now RUNNING
19/05/27 02:26:01 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20190527062557-0000/1 is now EXITED (Command exited with code 1)
19/05/27 02:26:01 INFO StandaloneSchedulerBackend: Executor app-20190527062557-0000/1 removed: Command exited with code 1
19/05/27 02:26:01 INFO BlockManagerMasterEndpoint: Trying to remove executor 1 from BlockManagerMaster.
19/05/27 02:26:01 INFO BlockManagerMaster: Removal of executor 1 requested
19/05/27 02:26:01 INFO CoarseGrainedSchedulerBackend$DriverEndpoint: Asked to remove non-existent executor 1
19/05/27 02:26:01 INFO StandaloneAppClient$ClientEndpoint: Executor added: app-20190527062557-0000/2 on worker-20190527062533-172.18.0.3-44351 (172.18.0.3:44351) with 1 core(s)
19/05/27 02:26:01 INFO StandaloneSchedulerBackend: Granted executor ID app-20190527062557-0000/2 on hostPort 172.18.0.3:44351 with 1 core(s), 1024.0 MB RAM
19/05/27 02:26:01 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20190527062557-0000/2 is now RUNNING
19/05/27 02:26:03 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20190527062557-0000/2 is now EXITED (Command exited with code 1)
from the worker container I see the job fails with the following exception
Caused by: java.io.IOException: Failed to connect to DESKTOP-PCH5L6D:53511
at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:245)
at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:187)
at org.apache.spark.rpc.netty.NettyRpcEnv.createClient(NettyRpcEnv.scala:198)
at org.apache.spark.rpc.netty.Outbox$$anon$1.call(Outbox.scala:194)
at org.apache.spark.rpc.netty.Outbox$$anon$1.call(Outbox.scala:190)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.UnknownHostException: DESKTOP-PCH5L6D
at java.net.InetAddress.getAllByName0(InetAddress.java:1281)
at java.net.InetAddress.getAllByName(InetAddress.java:1193)
at java.net.InetAddress.getAllByName(InetAddress.java:1127)
at java.net.InetAddress.getByName(InetAddress.java:1077)
added the additional parameter extra_hosts to fix the issue as shown below
version: "3.7"
services:
spark-master:
image: cmishr4/spark:latest
container_name: spark-master
hostname: spark-master
ports:
- "8080:8080"
- "7077:7077"
volumes:
- ./../share:/share
privileged: true
networks:
- spark-network
environment:
- "SPARK_MASTER_PORT=7077"
- "SPARK_MASTER_WEBUI_PORT=8080"
command: "sh start-master.sh"
spark-worker:
image: cmishr4/spark:latest
depends_on:
- spark-master
ports:
- 8080
volumes:
- ./../share:/share
privileged: true
networks:
- spark-network
extra_hosts:
- "DESKTOP-PCH5L6D:10.0.75.1"
environment:
- "SPARK_MASTER=spark://spark-master:7077"
- "SPARK_WORKER_WEBUI_PORT=8080"
command: "sh start-worker.sh"
networks:
spark-network:
driver: bridge
ipam:
driver: default
I've already installed a kubernetes docker for desktop and it's working fine. Then, I installed and configured a jenkins container.
Now, I want to deploy from jenkins container to kubernetes.
I installed Kubernetes Continuous Deploy Plugin and I configured the credential using the result of "kubectl config view" command
apiVersion: v1
clusters:
- cluster:
insecure-skip-tls-verify: true
server: https://localhost:6445
name: docker-for-desktop-cluster
contexts:
- context:
cluster: docker-for-desktop-cluster
user: docker-for-desktop
name: docker-for-desktop
current-context: docker-for-desktop
kind: Config
preferences: {}
When I try to deploy to kubernetes from jenkins, I get this error
ERROR: ERROR: io.fabric8.kubernetes.client.KubernetesClientException: Operation: [get] for kind: [Deployment] with name: [sudoku] in namespace: [default] failed.
hudson.remoting.ProxyException: io.fabric8.kubernetes.client.KubernetesClientException: Operation: [get] for kind: [Deployment] with name: [sudoku] in namespace: [default] failed.
at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:62)
at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:71)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:206)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:162)
at com.microsoft.jenkins.kubernetes.KubernetesClientWrapper$DeploymentUpdater.getCurrentResource(KubernetesClientWrapper.java:404)
at com.microsoft.jenkins.kubernetes.KubernetesClientWrapper$DeploymentUpdater.getCurrentResource(KubernetesClientWrapper.java:392)
at com.microsoft.jenkins.kubernetes.KubernetesClientWrapper$ResourceUpdater.createOrApply(KubernetesClientWrapper.java:358)
at com.microsoft.jenkins.kubernetes.KubernetesClientWrapper.apply(KubernetesClientWrapper.java:157)
at com.microsoft.jenkins.kubernetes.command.DeploymentCommand$DeploymentTask.doCall(DeploymentCommand.java:168)
at com.microsoft.jenkins.kubernetes.command.DeploymentCommand$DeploymentTask.call(DeploymentCommand.java:122)
at com.microsoft.jenkins.kubernetes.command.DeploymentCommand$DeploymentTask.call(DeploymentCommand.java:105)
at hudson.FilePath.act(FilePath.java:1165)
at com.microsoft.jenkins.kubernetes.command.DeploymentCommand.execute(DeploymentCommand.java:67)
at com.microsoft.jenkins.kubernetes.command.DeploymentCommand.execute(DeploymentCommand.java:46)
at com.microsoft.jenkins.azurecommons.command.CommandService.runCommand(CommandService.java:88)
at com.microsoft.jenkins.azurecommons.command.CommandService.execute(CommandService.java:96)
at com.microsoft.jenkins.azurecommons.command.CommandService.executeCommands(CommandService.java:75)
at com.microsoft.jenkins.azurecommons.command.BaseCommandContext.executeCommands(BaseCommandContext.java:77)
at com.microsoft.jenkins.kubernetes.KubernetesDeploy.perform(KubernetesDeploy.java:42)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.maven.MavenModuleSetBuild$MavenModuleSetBuildExecution.build(MavenModuleSetBuild.java:945)
at hudson.maven.MavenModuleSetBuild$MavenModuleSetBuildExecution.doRun(MavenModuleSetBuild.java:896)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:504)
at hudson.model.Run.execute(Run.java:1810)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.remoting.ProxyException: java.net.ConnectException: Failed to connect to /localhost:6445
at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.java:240)
at okhttp3.internal.connection.RealConnection.connect(RealConnection.java:158)
at okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:256)
at okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:134)
at okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:113)
at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:125)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
at io.fabric8.kubernetes.client.utils.ImpersonatorInterceptor.intercept(ImpersonatorInterceptor.java:54)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
at io.fabric8.kubernetes.client.utils.HttpClientUtils$2.intercept(HttpClientUtils.java:95)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:200)
at okhttp3.RealCall.execute(RealCall.java:77)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:379)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:344)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:313)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:296)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleGet(BaseOperation.java:770)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:195)
... 26 more
Caused by: hudson.remoting.ProxyException: java.net.ConnectException: Connection refused (Connection refused)
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at okhttp3.internal.platform.Platform.connectSocket(Platform.java:125)
at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.java:238)
... 55 more
Obviusly the error is the address of kubernetes but I don't know how to retrieve the correct one.
I tried the one I can see in kubernetes under the Service > Kubernetes > Deploy but it doen't work. Where can I find the correct one?
Your problem lays in kubeconfig file, to be precise in following section:
server: https://localhost:6445
It`s of course valid in context of your local machine, but not when you are trying to use it to reach Kubernetes cluster from inside of node/agent jenkins container. Containers are having unique hostname, so talking from container to https://localhost address is like talking to itself.
Solution:
please replace 'localhost' with either 'host.docker.internal', 'docker.for.win.localhost' (if you are on Windows) or simply local IP address of your local machine, like this:
apiVersion: v1
clusters:
- cluster:
insecure-skip-tls-verify: true
server: https://docker.for.win.localhost:6445
name: docker-for-desktop-cluster
contexts:
- context:
cluster: docker-for-desktop-cluster
user: docker-for-desktop
name: docker-for-desktop
current-context: docker-for-desktop
...
You can read more on Docker Desktop networking here
I'm need up Kafka and Cassandra in Minikube
Host OS is Ubuntu 16.04
$ uname -a
Linux minikuber 4.4.0-31-generic #50-Ubuntu SMP Wed Jul 13 00:07:12 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux
Minikube started normally:
$ minikube start
Starting local Kubernetes v1.8.0 cluster...
Starting VM...
Getting VM IP address...
Moving files into cluster...
Setting up certs...
Connecting to cluster...
Setting up kubeconfig...
Starting cluster components...
Kubectl is now configured to use the cluster.
Services list:
$ kubectl get services
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.0.0.1 <none> 443/TCP 1d
Zookeeper and Cassandra is running, but kafka crashing with error "CrashLoopBackOff"
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
zookeeper-775db4cd8-lpl95 1/1 Running 0 1h
cassandra-d84d697b8-p5wcs 1/1 Running 0 1h
kafka-6d889c567-w5n4s 0/1 CrashLoopBackOff 25 1h
View logs:
kubectl logs kafka-6d889c567-w5n4s -p
Output:
waiting for kafka to be ready
...
INFO Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) (org.apache.zookeeper.ClientCnxn)
INFO Waiting for keeper state SyncConnected (org.I0Itec.zkclient.ZkClient)
WARN Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect (org.apache.zookeeper.ClientCnxn)
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
INFO Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) (org.apache.zookeeper.ClientCnxn)
WARN Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect (org.apache.zookeeper.ClientCnxn)
...
INFO Terminate ZkClient event thread. (org.I0Itec.zkclient.ZkEventThread)
INFO Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) (org.apache.zookeeper.ClientCnxn)
INFO Session: 0x0 closed (org.apache.zookeeper.ZooKeeper)
INFO EventThread shut down for session: 0x0 (org.apache.zookeeper.ClientCnxn)
FATAL Fatal error during KafkaServer startup. Prepare to shutdown (kafka.server.KafkaServer)
org.I0Itec.zkclient.exception.ZkTimeoutException: Unable to connect to zookeeper server '' with timeout of 6000 ms
...
INFO shutting down (kafka.server.KafkaServer)
INFO shut down completed (kafka.server.KafkaServer)
FATAL Exiting Kafka. (kafka.server.KafkaServerStartable)
Сan any one help how to solve the problem of restarting the container?
kubectl describe pod kafka-6d889c567-w5n4s
Output describe:
Name: kafka-6d889c567-w5n4s
Namespace: default
Node: minikube/192.168.99.100
Start Time: Thu, 23 Nov 2017 17:03:20 +0300
Labels: pod-template-hash=284457123
run=kafka
Annotations: kubernetes.io/created-by={"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"ReplicaSet","namespace":"default","name":"kafka-6d889c567","uid":"0fa94c8d-d057-11e7-ad48-080027a5dfed","a...
Status: Running
IP: 172.17.0.5
Created By: ReplicaSet/kafka-6d889c567
Controlled By: ReplicaSet/kafka-6d889c567
Info about Containers:
Containers:
kafka:
Container ID: docker://7ed3de8ef2e3e665ba693186f5125c6802283e1fabca8f3c85eb584f8de19526
Image: wurstmeister/kafka
Image ID: docker-pullable://wurstmeister/kafka#sha256:2aa183fd201d693e24d4d5d483b081fc2c62c198a7acb8484838328c83542c96
Port: <none>
State: Waiting
Reason: CrashLoopBackOff
Last State: Terminated
Reason: Error
Exit Code: 1
Started: Mon, 27 Nov 2017 09:43:39 +0300
Finished: Mon, 27 Nov 2017 09:43:49 +0300
Ready: False
Restart Count: 1003
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from default-token-bnz99 (ro)
Info about Conditions:
Conditions:
Type Status
Initialized True
Ready False
PodScheduled True
Info about volumes:
Volumes:
default-token-bnz99:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-bnz99
Optional: false
QoS Class: BestEffort
Info about events:
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Pulling 38m (x699 over 2d) kubelet, minikube pulling image "wurstmeister/kafka"
Warning BackOff 18m (x16075 over 2d) kubelet, minikube Back-off restarting failed container
Warning FailedSync 3m (x16140 over 2d) kubelet, minikube Error syncing pod
I am trying to run docker image of sonarqube with mysql db by below dockercommand:
sudo docker run -d --name hg-sonarqube \
-p 9000:9000 \
-e SONARQUBE_JDBC_USERNAME='sonar' \
-e SONARQUBE_JDBC_PASSWORD='sonar' \
-e SONARQUBE_JDBC_URL='jdbc:mysql://localhost:3306/sonar?useUnicode=true&characterEncoding=utf8&rewriteBatchedStatements=true&useConfigs=maxPerformance' \
sonarqube
But container is not running due to error:
2016.12.28 11:20:11 INFO web[][o.sonar.db.Database] Create JDBC data source for jdbc:mysql://localhost:3306/sonar?useUnicode=true&characterEncoding=utf8&rewriteBatchedStatements=true&useConfigs=maxPerformance
2016.12.28 11:20:11 ERROR web[][o.a.c.c.C.[.[.[/]] Exception sending context initialized event to listener instance of class org.sonar.server.platform.web.PlatformServletContextListener
java.lang.IllegalStateException: Can not connect to database. Please check connectivity and settings (see the properties prefixed by 'sonar.jdbc.').
at org.sonar.db.DefaultDatabase.checkConnection(DefaultDatabase.java:108)
MySQL service is running and sonar database. I use command to create database and give privileges in Ubuntu-14.04.
echo "GRANT ALL PRIVILEGES ON *.* TO 'root'#'%' IDENTIFIED BY 'welcome123'; flush privileges;" | mysql -u root -pwelcome123
echo "CREATE DATABASE sonar CHARACTER SET utf8 COLLATE utf8_general_ci; CREATE USER 'sonar' IDENTIFIED BY 'sonar';GRANT ALL PRIVILEGES ON sonar.* TO 'sonar'#'%' IDENTIFIED BY 'sonar'; GRANT ALL ON sonar.* TO 'sonar'#'localhost' IDENTIFIED BY 'sonar'; flush privileges;" | mysql -u root -pwelcome123
Full Log file:
2016.12.28 11:19:58 INFO app[][o.s.a.AppFileSystem] Cleaning or creating temp directory /opt/sonarqube/temp
2016.12.28 11:19:58 INFO app[][o.s.p.m.JavaProcessLauncher] Launch process[es]: /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java -Djava.awt.headless=true -Xmx1G -Xms256m -Xss256k -Djna.nosys=true -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -Djava.io.tmpdir=/opt/sonarqube/temp -javaagent:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/management-agent.jar -cp ./lib/common/*:./lib/search/* org.sonar.search.SearchServer /opt/sonarqube/temp/sq-process5713024831851311243properties
2016.12.28 11:19:59 INFO es[][o.s.p.ProcessEntryPoint] Starting es
2016.12.28 11:19:59 INFO es[][o.s.s.EsSettings] Elasticsearch listening on /127.0.0.1:9001
2016.12.28 11:19:59 INFO es[][o.elasticsearch.node] [sonarqube] version[2.3.5], pid[18], build[90f439f/2016-07-27T10:36:52Z]
2016.12.28 11:19:59 INFO es[][o.elasticsearch.node] [sonarqube] initializing ...
2016.12.28 11:19:59 INFO es[][o.e.plugins] [sonarqube] modules [], plugins [], sites []
2016.12.28 11:19:59 INFO es[][o.elasticsearch.env] [sonarqube] using [1] data paths, mounts [[/opt/sonarqube/data (/dev/sda1)]], net usable_space [24.2gb], net total_space [28.8gb], spins? [possibly], types [ext4]
2016.12.28 11:19:59 INFO es[][o.elasticsearch.env] [sonarqube] heap size [1007.3mb], compressed ordinary object pointers [true]
2016.12.28 11:20:03 INFO es[][o.elasticsearch.node] [sonarqube] initialized
2016.12.28 11:20:03 INFO es[][o.elasticsearch.node] [sonarqube] starting ...
2016.12.28 11:20:03 INFO es[][o.e.transport] [sonarqube] publish_address {127.0.0.1:9001}, bound_addresses {127.0.0.1:9001}
2016.12.28 11:20:03 INFO es[][o.e.discovery] [sonarqube] sonarqube/CPgnfx6NTe2aO07d6fR0Bg
2016.12.28 11:20:06 INFO es[][o.e.cluster.service] [sonarqube] new_master {sonarqube}{CPgnfx6NTe2aO07d6fR0Bg}{127.0.0.1}{127.0.0.1:9001}{rack_id=sonarqube, master=true}, reason: zen-disco-join(elected_as_master, [0] joins received)
2016.12.28 11:20:06 INFO es[][o.elasticsearch.node] [sonarqube] started
2016.12.28 11:20:06 INFO es[][o.e.gateway] [sonarqube] recovered [0] indices into cluster_state
2016.12.28 11:20:06 INFO app[][o.s.p.m.Monitor] Process[es] is up
2016.12.28 11:20:06 INFO app[][o.s.p.m.JavaProcessLauncher] Launch process[web]: /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djruby.management.enabled=false -Djruby.compile.invokedynamic=false -Xmx512m -Xms128m -XX:+HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Djava.io.tmpdir=/opt/sonarqube/temp -javaagent:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/management-agent.jar -cp ./lib/common/*:./lib/server/*:/opt/sonarqube/lib/jdbc/mysql/mysql-connector-java-5.1.39.jar org.sonar.server.app.WebServer /opt/sonarqube/temp/sq-process6242669754365841464properties
2016.12.28 11:20:08 INFO web[][o.s.p.ProcessEntryPoint] Starting web
2016.12.28 11:20:08 INFO web[][o.s.s.a.TomcatContexts] Webapp directory: /opt/sonarqube/web
2016.12.28 11:20:08 INFO web[][o.a.c.h.Http11NioProtocol] Initializing ProtocolHandler ["http-nio-0.0.0.0-9000"]
2016.12.28 11:20:08 INFO web[][o.a.t.u.n.NioSelectorPool] Using a shared selector for servlet write/read
2016.12.28 11:20:09 INFO web[][o.e.plugins] [Bushwacker] modules [], plugins [], sites []
2016.12.28 11:20:11 INFO web[][o.s.s.e.EsClientProvider] Connected to local Elasticsearch: [127.0.0.1:9001]
2016.12.28 11:20:11 INFO web[][o.s.s.p.LogServerVersion] SonarQube Server / 6.2 / 4a28f29f95254b58f3cf0a0871bc632e998403f5
2016.12.28 11:20:11 INFO web[][o.sonar.db.Database] Create JDBC data source for jdbc:mysql://localhost:3306/sonar?useUnicode=true&characterEncoding=utf8&rewriteBatchedStatements=true&useConfigs=maxPerformance
2016.12.28 11:20:11 ERROR web[][o.a.c.c.C.[.[.[/]] Exception sending context initialized event to listener instance of class org.sonar.server.platform.web.PlatformServletContextListener
java.lang.IllegalStateException: Can not connect to database. Please check connectivity and settings (see the properties prefixed by 'sonar.jdbc.').
at org.sonar.db.DefaultDatabase.checkConnection(DefaultDatabase.java:108)
at org.sonar.db.DefaultDatabase.start(DefaultDatabase.java:75)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.picocontainer.lifecycle.ReflectionLifecycleStrategy.invokeMethod(ReflectionLifecycleStrategy.java:110)
at org.picocontainer.lifecycle.ReflectionLifecycleStrategy.start(ReflectionLifecycleStrategy.java:89)
at org.sonar.core.platform.ComponentContainer$1.start(ComponentContainer.java:320)
at org.picocontainer.injectors.AbstractInjectionFactory$LifecycleAdapter.start(AbstractInjectionFactory.java:84)
at org.picocontainer.behaviors.AbstractBehavior.start(AbstractBehavior.java:169)
at org.picocontainer.behaviors.Stored$RealComponentLifecycle.start(Stored.java:132)
at org.picocontainer.behaviors.Stored.start(Stored.java:110)
at org.picocontainer.DefaultPicoContainer.potentiallyStartAdapter(DefaultPicoContainer.java:1016)
at org.picocontainer.DefaultPicoContainer.startAdapters(DefaultPicoContainer.java:1009)
at org.picocontainer.DefaultPicoContainer.start(DefaultPicoContainer.java:767)
at org.sonar.core.platform.ComponentContainer.startComponents(ComponentContainer.java:141)
at org.sonar.server.platform.platformlevel.PlatformLevel.start(PlatformLevel.java:88)
at org.sonar.server.platform.Platform.start(Platform.java:216)
at org.sonar.server.platform.Platform.startLevel1Container(Platform.java:175)
at org.sonar.server.platform.Platform.init(Platform.java:90)
at org.sonar.server.platform.web.PlatformServletContextListener.contextInitialized(PlatformServletContextListener.java:44)
at org.apache.catalina.core.StandardContext.listenerStart(StandardContext.java:4812)
at org.apache.catalina.core.StandardContext.startInternal(StandardContext.java:5255)
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:147)
at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1408)
at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1398)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.commons.dbcp.SQLNestedException: Cannot create PoolableConnectionFactory (Communications link failure
The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.)
at org.apache.commons.dbcp.BasicDataSource.createPoolableConnectionFactory(BasicDataSource.java:1549)
at org.apache.commons.dbcp.BasicDataSource.createDataSource(BasicDataSource.java:1388)
at org.apache.commons.dbcp.BasicDataSource.getConnection(BasicDataSource.java:1044)
at org.sonar.db.profiling.NullConnectionInterceptor.getConnection(NullConnectionInterceptor.java:31)
at org.sonar.db.profiling.ProfiledDataSource.getConnection(ProfiledDataSource.java:323)
at org.sonar.db.DefaultDatabase.checkConnection(DefaultDatabase.java:106)
... 30 common frames omitted
Caused by: com.mysql.jdbc.exceptions.jdbc4.CommunicationsException: Communications link failure
The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
Might be helpful for those who still facing this issue
You might be running mysql in another container. You can use below docker-compose file to make it in the same network
# Use sonar/sonar as user/password credentials
version: '3.1'
services:
sonarqube:
image: sonarqube:5.1.1
networks:
- sonarqube-network
ports:
- "9000:9000"
- "3306:3306"
environment:
- SONARQUBE_JDBC_USERNAME=sonar
- SONARQUBE_JDBC_PASSWORD=sonar
- SONARQUBE_JDBC_URL=jdbc:mysql://localhost:3306/sonar?useUnicode=true&characterEncoding=utf8&rewriteBatchedStatements=true
db:
image: mysql
networks:
- sonarqube-network
environment:
- MYSQL_ROOT_PASSWORD=sonar
- MYSQL_DATABASE=sonar
- MYSQL_USER=sonar
- MYSQL_PASSWORD=sonar
networks:
sonarqube-network:
Save the file as docker-compose.yml and run docker-compose up
Please note this entry,
- 3306:3306
After that try with,
mysql -u sonar -h localhost -p
to connect to MySQL