kafka consumption through mapreduce or yarn failing with connection refused error on hortonworks secure cluster. The destination host is getting displayed as
No where i am providing this details. The destination work dir has metrics and working directory created as well. Failing when creating the output directory i guess. The standard gobblin application.conf and reference.conf files with all the necessary parameters is placed.
.
WARN [NativeCodeLoader] Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
WARN [BlockReaderLocal] The short-circuit local reads feature cannot be used because libhadoop cannot be loaded.
WARN [ZKHelixAdmin] Root directory exists.Cleaning the root directory:/GobblinYarnVin
WARN [RetryInvocationHandler] Exception while invoking class org.apache.hadoop.yarn.api.impl.pb.client.ApplicationClientProtocolPBClientImpl.getApplications. Not retrying because failovers (30) exceeded maximum allowed (30)
at sun.reflect.GeneratedConstructorAccessor5.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:783)
at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:730)
at org.apache.hadoop.ipc.Client.call(Client.java:1410)
at org.apache.hadoop.ipc.Client.call(Client.java:1359)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
at com.sun.proxy.$Proxy19.getApplications(Unknown Source)
at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationClientProtocolPBClientImpl.getApplications(ApplicationClientProtocolPBClientImpl.java:197)
at sun.reflect.GeneratedMethodAccessor1.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:186)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy20.getApplications(Unknown Source)
at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.getApplications(YarnClientImpl.java:285)
at gobblin.yarn.GobblinYarnAppLauncher.getReconnectableApplicationId(GobblinYarnAppLauncher.java:434)
at gobblin.yarn.GobblinYarnAppLauncher.getApplicationId(GobblinYarnAppLauncher.java:421)
at gobblin.yarn.GobblinYarnAppLauncher.launch(GobblinYarnAppLauncher.java:254)
at gobblin.yarn.GobblinYarnAppLauncher.main(GobblinYarnAppLauncher.java:794)
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:529)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:493)
at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:601)
at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:696)
at org.apache.hadoop.ipc.Client$Connection.access$2700(Client.java:367)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1458)
at org.apache.hadoop.ipc.Client.call(Client.java:1377)
... 15 more
bash-4.1$