Hi,
Getting the error in gobblin MRlauncher job. Please help
LOGS :
1.
http://localhost:50070/logs/userlogs/application_1496818923605_0001/container_1496818923605_0001_01_000001/syslog2017-06-07 12:40:32,978 ERROR [IPC Server handler 19 on 34746] org.apache.hadoop.mapred.TaskAttemptListenerImpl: Task: attempt_1496818923605_0001_m_000001_2 - exited : java.io.IOException: Not all tasks running in container attempt_1496818923605_0001_m_000001_2 completed successfully
at gobblin.runtime.GobblinMultiTaskAttempt.persistTaskStateStore(GobblinMultiTaskAttempt.java:246)
at gobblin.runtime.GobblinMultiTaskAttempt.commit(GobblinMultiTaskAttempt.java:183)
at gobblin.runtime.GobblinMultiTaskAttempt.runAndOptionallyCommitTaskAttempt(GobblinMultiTaskAttempt.java:330)
at gobblin.runtime.GobblinMultiTaskAttempt.runWorkUnits(GobblinMultiTaskAttempt.java:378)
at gobblin.runtime.mapreduce.MRJobLauncher$TaskRunner.run(MRJobLauncher.java:636)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:175)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1807)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:169)
2017-06-07 12:40:32,978 INFO [IPC Server handler 19 on 34746] org.apache.hadoop.mapred.TaskAttemptListenerImpl: Diagnostics report from attempt_1496818923605_0001_m_000001_2: Error: java.io.IOException: Not all tasks running in container attempt_1496818923605_0001_m_000001_2 completed successfully
at gobblin.runtime.GobblinMultiTaskAttempt.persistTaskStateStore(GobblinMultiTaskAttempt.java:246)
at gobblin.runtime.GobblinMultiTaskAttempt.commit(GobblinMultiTaskAttempt.java:183)
at gobblin.runtime.GobblinMultiTaskAttempt.runAndOptionallyCommitTaskAttempt(GobblinMultiTaskAttempt.java:330)
at gobblin.runtime.GobblinMultiTaskAttempt.runWorkUnits(GobblinMultiTaskAttempt.java:378)
at gobblin.runtime.mapreduce.MRJobLauncher$TaskRunner.run(MRJobLauncher.java:636)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:175)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1807)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:169)
2017-06-07 12:40:32,978 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl: Diagnostics report from attempt_1496818923605_0001_m_000001_2: Error: java.io.IOException: Not all tasks running in container attempt_1496818923605_0001_m_000001_2 completed successfully
at gobblin.runtime.GobblinMultiTaskAttempt.persistTaskStateStore(GobblinMultiTaskAttempt.java:246)
at gobblin.runtime.GobblinMultiTaskAttempt.commit(GobblinMultiTaskAttempt.java:183)
at gobblin.runtime.GobblinMultiTaskAttempt.runAndOptionallyCommitTaskAttempt(GobblinMultiTaskAttempt.java:330)
at gobblin.runtime.GobblinMultiTaskAttempt.runWorkUnits(GobblinMultiTaskAttempt.java:378)
at gobblin.runtime.mapreduce.MRJobLauncher$TaskRunner.run(MRJobLauncher.java:636)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:175)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1807)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:169)
2. gobblin-current.log
2017-06-07 12:41:11 IST ERROR [main] gobblin.util.executors.IteratorExecutor 163 - Iterator executor failure.
java.util.concurrent.ExecutionException: java.lang.RuntimeException: Not committing dataset of job job_kafkaHdfs_1496819289443 with commit policy
COMMIT_ON_FULL_SUCCESS and state FAILED
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at gobblin.util.executors.IteratorExecutor.executeAndGetResults(IteratorExecutor.java:128)
at gobblin.runtime.JobContext.commit(JobContext.java:450)
at gobblin.runtime.JobContext.commit(JobContext.java:416)
at gobblin.runtime.AbstractJobLauncher.launchJob(AbstractJobLauncher.java:433)
at gobblin.runtime.mapreduce.CliMRJobLauncher.launchJob(CliMRJobLauncher.java:89)
at gobblin.runtime.mapreduce.CliMRJobLauncher.run(CliMRJobLauncher.java:66)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:90)
at gobblin.runtime.mapreduce.CliMRJobLauncher.main(CliMRJobLauncher.java:111)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.util.RunJar.run(RunJar.java:234)
at org.apache.hadoop.util.RunJar.main(RunJar.java:148)
Caused by: java.lang.RuntimeException: Not committing dataset of job job_kafkaHdfs_1496819289443 with commit policy COMMIT_ON_FULL_SUCCESS and state FAILED
at gobblin.runtime.SafeDatasetCommit.call(SafeDatasetCommit.java:86)
at gobblin.runtime.SafeDatasetCommit.call(SafeDatasetCommit.java:54)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at gobblin.util.executors.MDCPropagatingRunnable.run(MDCPropagatingRunnable.java:35)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
2017-06-07 12:41:11 IST ERROR [main] gobblin.runtime.AbstractJobLauncher 444 - Failed to launch and run job job_kafkaHdfs_1496819289443: java.io.IOException: Failed to commit dataset state for some dataset(s) of job job_kafkaHdfs_1496819289443
java.io.IOException: Failed to commit dataset state for some dataset(s) of job job_kafkaHdfs_1496819289443
at gobblin.runtime.JobContext.commit(JobContext.java:456)
at gobblin.runtime.JobContext.commit(JobContext.java:416)
at gobblin.runtime.AbstractJobLauncher.launchJob(AbstractJobLauncher.java:433)
at gobblin.runtime.mapreduce.CliMRJobLauncher.launchJob(CliMRJobLauncher.java:89)
at gobblin.runtime.mapreduce.CliMRJobLauncher.run(CliMRJobLauncher.java:66)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
and config is
job.name=kafkaHdfs
job.group=kh-group
job.description=kafka to hdfs
job.lock.enabled=false
job.schedule=0 0/2 * * * ?
launcher.type=MAPREDUCE
source.class=com.test.gobblin.KafkaCustomSource
converter.classes=com.test.gobblin.KafkaCustomConverter
converter.avro.binary.charset=UTF-16
extract.namespace=kafka-gobblin-hdfs
mr.job.max.mappers=2
mr.job.root.dir=hdfs://hadoop-master:8089/gobblin/root
workunit.retry.enabled=true
workunit.retry.policy=always
bootstrap.with.offset=earliest
kafka.brokers=kafka.server.com:9092
topic.name=gob-hdfs
topic.whitelist=gob-hdfs
state.store.fs.uri=hdfs://hadoop-master:8089/
state.store.dir=/gobblin/state
fs.uri=hdfs://hadoop-master:8089/
writer.fs.uri=hdfs://hadoop-master:8089/
writer.destination.type=HDFS
writer.output.format=AVRO
writer.output.schema={"namespace":"kafka-gobblin-hdfs", "type":"record", "name":"event", "fields":[{"name":"eventId", "type":"long"}, {"name":"eventItemId", "type":"string"}, {"name":"logTime", "type":"long"}, {"name":"itemStatus", "type":"string"}]}
writer.staging.dir=/gobblin/writer/staging
writer.output.dir=/gobblin/writer/output
data.publisher.final.dir=/gobblin/output
data.publisher.type=gobblin.publisher.BaseDataPublisher