Rmr2 java.lang.RuntimeException: PipeMapRed.waitOutputThreads(): subprocess failed with code 1

1,014 views
Skip to first unread message

Ved J.

unread,
Aug 2, 2014, 3:08:22 PM8/2/14
to rha...@googlegroups.com
Can someone please help with this?

I have a 4 node CDH5 ubuntu 12.04 cluster with R 3.1.1

I am trying to run this:

ints = to.dfs(1:10)
calc = mapreduce(input=ints, map=function(k,v) cbind(v,v*2))

But I get the following error:
Error: java.lang.RuntimeException: PipeMapRed.waitOutputThreads(): subprocess failed with code 1
        at org.apache.hadoop.streaming.PipeMapRed.waitOutputThreads(PipeMapRed.java:320)
        at org.apache.hadoop.streaming.PipeMapRed.mapRedFinished(PipeMapRed.java:533)
        at org.apache.hadoop.streaming.PipeMapper.close(PipeMapper.java:130)
        at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:61)
        at org.apache.hadoop.streaming.PipeMapRunner.run(PipeMapRunner.java:34)
        at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:430)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)

14/08/02 19:05:34 INFO mapreduce.Job: Task Id : attempt_1407004891379_0005_m_000000_0, Status : FAILED
Error: java.lang.RuntimeException: PipeMapRed.waitOutputThreads(): subprocess failed with code 1
        at org.apache.hadoop.streaming.PipeMapRed.waitOutputThreads(PipeMapRed.java:320)
        at org.apache.hadoop.streaming.PipeMapRed.mapRedFinished(PipeMapRed.java:533)
        at org.apache.hadoop.streaming.PipeMapper.close(PipeMapper.java:130)
        at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:61)
        at org.apache.hadoop.streaming.PipeMapRunner.run(PipeMapRunner.java:34)
        at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:430)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)

14/08/02 19:05:43 INFO mapreduce.Job: Task Id : attempt_1407004891379_0005_m_000000_1, Status : FAILED
Error: java.lang.RuntimeException: PipeMapRed.waitOutputThreads(): subprocess failed with code 1
        at org.apache.hadoop.streaming.PipeMapRed.waitOutputThreads(PipeMapRed.java:320)
        at org.apache.hadoop.streaming.PipeMapRed.mapRedFinished(PipeMapRed.java:533)
        at org.apache.hadoop.streaming.PipeMapper.close(PipeMapper.java:130)
        at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:61)
        at org.apache.hadoop.streaming.PipeMapRunner.run(PipeMapRunner.java:34)
        at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:430)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)

14/08/02 19:05:44 INFO mapreduce.Job: Task Id : attempt_1407004891379_0005_m_000001_1, Status : FAILED
Error: java.lang.RuntimeException: PipeMapRed.waitOutputThreads(): subprocess failed with code 1
        at org.apache.hadoop.streaming.PipeMapRed.waitOutputThreads(PipeMapRed.java:320)
        at org.apache.hadoop.streaming.PipeMapRed.mapRedFinished(PipeMapRed.java:533)
        at org.apache.hadoop.streaming.PipeMapper.close(PipeMapper.java:130)
        at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:61)
        at org.apache.hadoop.streaming.PipeMapRunner.run(PipeMapRunner.java:34)
        at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:430)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)

14/08/02 19:05:49 INFO mapreduce.Job: Task Id : attempt_1407004891379_0005_m_000000_2, Status : FAILED
Error: java.lang.RuntimeException: PipeMapRed.waitOutputThreads(): subprocess failed with code 1
        at org.apache.hadoop.streaming.PipeMapRed.waitOutputThreads(PipeMapRed.java:320)
        at org.apache.hadoop.streaming.PipeMapRed.mapRedFinished(PipeMapRed.java:533)
        at org.apache.hadoop.streaming.PipeMapper.close(PipeMapper.java:130)
        at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:61)
        at org.apache.hadoop.streaming.PipeMapRunner.run(PipeMapRunner.java:34)
        at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:430)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)

14/08/02 19:05:50 INFO mapreduce.Job: Task Id : attempt_1407004891379_0005_m_000001_2, Status : FAILED
Error: java.lang.RuntimeException: PipeMapRed.waitOutputThreads(): subprocess failed with code 1
        at org.apache.hadoop.streaming.PipeMapRed.waitOutputThreads(PipeMapRed.java:320)
        at org.apache.hadoop.streaming.PipeMapRed.mapRedFinished(PipeMapRed.java:533)
        at org.apache.hadoop.streaming.PipeMapper.close(PipeMapper.java:130)
        at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:61)
        at org.apache.hadoop.streaming.PipeMapRunner.run(PipeMapRunner.java:34)
        at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:430)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:342)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)

14/08/02 19:05:57 INFO mapreduce.Job:  map 100% reduce 0%
14/08/02 19:05:57 INFO mapreduce.Job: Job job_1407004891379_0005 failed with state FAILED due to: Task failed task_1407004891379_0005_m_000000
Job failed as tasks failed. failedMaps:1 failedReduces:0

14/08/02 19:05:57 INFO mapreduce.Job: Counters: 13
        Job Counters
                Failed map tasks=7
                Killed map tasks=1
                Launched map tasks=8
                Other local map tasks=6
                Data-local map tasks=2
                Total time spent by all maps in occupied slots (ms)=40116
                Total time spent by all reduces in occupied slots (ms)=0
                Total time spent by all map tasks (ms)=40116
                Total vcore-seconds taken by all map tasks=40116
                Total megabyte-seconds taken by all map tasks=41078784
        Map-Reduce Framework
                CPU time spent (ms)=0
                Physical memory (bytes) snapshot=0
                Virtual memory (bytes) snapshot=0
14/08/02 19:05:57 ERROR streaming.StreamJob: Job not Successful!
Streaming Command Failed!
Error in mr(map = map, reduce = reduce, combine = combine, vectorized.reduce,  :
  hadoop streaming failed with error code 1


I don't see anything more in the Resource manager logs.

Please help. I really need t ohave this working.

Thanks,
Vedant

Antonio Piccolboni

unread,
Aug 2, 2014, 3:15:30 PM8/2/14
to rha...@googlegroups.com
Did you read the intro message to this group? With this information, there's nothing I can do, unless I can reproduce your error (I can't). 



Antonio

Vedant Jain

unread,
Aug 2, 2014, 3:28:32 PM8/2/14
to rha...@googlegroups.com
Hi Antonio, 

Yes I did. But this is all the info I have. There is nothing in the log files. I looked at the Resource Manager and Job History logs but couldn't find anything more. 

Can you point me where I should be looking for the R stderr?

Thanks,
Ved


--
post: rha...@googlegroups.com ||
unsubscribe: rhadoop+u...@googlegroups.com ||
web: https://groups.google.com/d/forum/rhadoop?hl=en-US
---
You received this message because you are subscribed to a topic in the Google Groups "RHadoop" group.
To unsubscribe from this topic, visit https://groups.google.com/d/topic/rhadoop/bnesaAtyaPo/unsubscribe.
To unsubscribe from this group and all its topics, send an email to rhadoop+u...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

Antonio Piccolboni

unread,
Aug 2, 2014, 3:35:51 PM8/2/14
to RHadoop Google Group
Job history logs. I think this happened once before. Let me try to dig up the threads a bit.


You received this message because you are subscribed to the Google Groups "RHadoop" group.
To unsubscribe from this group and stop receiving emails from it, send an email to rhadoop+u...@googlegroups.com.

Vedant Jain

unread,
Aug 2, 2014, 3:35:57 PM8/2/14
to rha...@googlegroups.com
Antonio,

Here is what I found in stderr

 Log Type: stderr
  
  Log Length: 381
  Error in library(functional) : there is no package called ‘functional’
  No traceback available
  Error during wrapup:
  Execution halted
  log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.impl.MetricsSystemImpl).
  log4j:WARN Please initialize the log4j system properly.
  log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.


On Sat, Aug 2, 2014 at 2:15 PM, Antonio Piccolboni <picc...@gmail.com> wrote:

Antonio Piccolboni

unread,
Aug 2, 2014, 3:39:31 PM8/2/14
to RHadoop Google Group
Fantastic! Install functional everywhere. If it's already installed, make sure it's installed into a system path, not a user path. Repeat for all dependencies. Let me know if that gets you going.


You received this message because you are subscribed to the Google Groups "RHadoop" group.
To unsubscribe from this group and stop receiving emails from it, send an email to rhadoop+u...@googlegroups.com.

Vedant Jain

unread,
Aug 2, 2014, 5:22:18 PM8/2/14
to rha...@googlegroups.com
Thanks Antonio!!! It worked (:

Until next time ;)


14/08/02 21:20:20 INFO mapreduce.Job: Job job_1407004891379_0009 running in uber                                                                                                               mode : false
14/08/02 21:20:20 INFO mapreduce.Job:  map 0% reduce 0%
14/08/02 21:20:28 INFO mapreduce.Job:  map 50% reduce 0%
14/08/02 21:20:29 INFO mapreduce.Job:  map 100% reduce 0%
14/08/02 21:20:29 INFO mapreduce.Job: Job job_1407004891379_0009 completed succe                                                                                                              ssfully
14/08/02 21:20:30 INFO mapreduce.Job: Counters: 30
        File System Counters
                FILE: Number of bytes read=0
                FILE: Number of bytes written=194810
                FILE: Number of read operations=0
                FILE: Number of large read operations=0
                FILE: Number of write operations=0
                HDFS: Number of bytes read=1188
                HDFS: Number of bytes written=2379
                HDFS: Number of read operations=14
                HDFS: Number of large read operations=0
                HDFS: Number of write operations=4
        Job Counters
                Launched map tasks=2
                Data-local map tasks=2
                Total time spent by all maps in occupied slots (ms)=13385
                Total time spent by all reduces in occupied slots (ms)=0
                Total time spent by all map tasks (ms)=13385
                Total vcore-seconds taken by all map tasks=13385
                Total megabyte-seconds taken by all map tasks=13706240
        Map-Reduce Framework
                Map input records=3
                Map output records=3
                Input split bytes=200
                Spilled Records=0
                Failed Shuffles=0
                Merged Map outputs=0
                GC time elapsed (ms)=75
                CPU time spent (ms)=2850
                Physical memory (bytes) snapshot=343666688
                Virtual memory (bytes) snapshot=3367895040
                Total committed heap usage (bytes)=391118848
        File Input Format Counters
                Bytes Read=988
        File Output Format Counters
                Bytes Written=2379
14/08/02 21:20:30 INFO streaming.StreamJob: Output directory: /tmp/file493a1a6d4    
Reply all
Reply to author
Forward
0 new messages