2016-07-03 17:34:50,316 ERROR [main] cascading.flow.stream.TrapHandler: caught OutOfMemoryException, will not trap, rethrowing java.lang.OutOfMemoryError: GC overhead limit exceeded at com.twitter.chill.hadoop.KryoDeserializer.deserialize(KryoDeserializer.java:49) at cascading.tuple.hadoop.TupleSerialization$SerializationElementReader.read(TupleSerialization.java:628) at cascading.tuple.hadoop.io.HadoopTupleInputStream.readType(HadoopTupleInputStream.java:105) at cascading.tuple.hadoop.io.HadoopTupleInputStream.getNextElement(HadoopTupleInputStream.java:52) at cascading.tuple.hadoop.util.TupleElementComparator.compare(TupleElementComparator.java:78) at cascading.tuple.hadoop.util.TupleElementComparator.compare(TupleElementComparator.java:33) at cascading.tuple.hadoop.util.DeserializerComparator.compareTuples(DeserializerComparator.java:160) at cascading.tuple.hadoop.util.TupleComparator.compare(TupleComparator.java:38) at org.apache.hadoop.mapred.Merger$MergeQueue.lessThan(Merger.java:587) at org.apache.hadoop.util.PriorityQueue.downHeap(PriorityQueue.java:144) at org.apache.hadoop.util.PriorityQueue.adjustTop(PriorityQueue.java:108) at org.apache.hadoop.mapred.Merger$MergeQueue.adjustPriorityQueue(Merger.java:524) at org.apache.hadoop.mapred.Merger$MergeQueue.next(Merger.java:547) at org.apache.hadoop.mapred.Task$ValuesIterator.readNextKey(Task.java:1411) at org.apache.hadoop.mapred.Task$ValuesIterator.next(Task.java:1370) at org.apache.hadoop.mapred.ReduceTask$ReduceValuesIterator.moveToNext(ReduceTask.java:220) at org.apache.hadoop.mapred.ReduceTask$ReduceValuesIterator.next(ReduceTask.java:216) at cascading.flow.hadoop.util.TimedIterator.next(TimedIterator.java:74) at cascading.flow.hadoop.HadoopGroupByClosure$1.next(HadoopGroupByClosure.java:113) at cascading.flow.hadoop.HadoopGroupByClosure$1.next(HadoopGroupByClosure.java:71) at cascading.pipe.joiner.InnerJoin$JoinIterator.next(InnerJoin.java:190) at cascading.pipe.joiner.InnerJoin$JoinIterator.next(InnerJoin.java:68) at cascading.tuple.TupleEntryChainIterator.next(TupleEntryChainIterator.java:79) at cascading.tuple.TupleEntryChainIterator.next(TupleEntryChainIterator.java:32) at cascading.flow.stream.BufferEveryWindow$2.next(BufferEveryWindow.java:153) at cascading.flow.stream.BufferEveryWindow$2.next(BufferEveryWindow.java:140) at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:42) at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) at com.akamai.csi.jobs.needles.scalding.fpneedle.job.SelectorsIterator.next(SelectorsIterator.scala:49) at com.akamai.csi.jobs.needles.scalding.fpneedle.job.SelectorsIterator.next(SelectorsIterator.scala:8) at com.twitter.scalding.TypedBufferOp.operate(Operations.scala:499) at cascading.flow.stream.BufferEveryWindow.receive(BufferEveryWindow.java:125) 2016-07-03 17:34:57,587 FATAL [main] org.apache.hadoop.mapred.YarnChild: Error running child : java.lang.OutOfMemoryError: GC overhead limit exceeded at scala.collection.immutable.HashMap$HashTrieMap.updated0(HashMap.scala:328) at scala.collection.immutable.HashMap.$plus(HashMap.scala:57) at scala.collection.immutable.HashMap.$plus(HashMap.scala:36) at scala.collection.mutable.MapBuilder.$plus$eq(MapBuilder.scala:28) at scala.collection.mutable.MapBuilder.$plus$eq(MapBuilder.scala:24) at scala.collection.TraversableOnce$$anonfun$toMap$1.apply(TraversableOnce.scala:280) at scala.collection.TraversableOnce$$anonfun$toMap$1.apply(TraversableOnce.scala:279) at scala.collection.Iterator$class.foreach(Iterator.scala:727) at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) at scala.collection.TraversableOnce$class.toMap(TraversableOnce.scala:279) at scala.collection.AbstractIterator.toMap(Iterator.scala:1157) at com.twitter.algebird.SummingCache.flush(SummingCache.scala:61) at com.twitter.scalding.MapsideReduce.flush(Operations.scala:183) at cascading.flow.stream.OperatorStage.complete(OperatorStage.java:292) at cascading.flow.stream.Duct.complete(Duct.java:81) at cascading.flow.stream.OperatorStage.complete(OperatorStage.java:296) at cascading.flow.stream.Duct.complete(Duct.java:81) at cascading.flow.stream.OperatorStage.complete(OperatorStage.java:296) at cascading.flow.stream.Duct.complete(Duct.java:81) at cascading.flow.hadoop.stream.HadoopGroupGate.complete(HadoopGroupGate.java:77) at cascading.flow.hadoop.FlowReducer.close(FlowReducer.java:161) at org.apache.hadoop.io.IOUtils.cleanup(IOUtils.java:244) at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:459) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:392) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
"tez.task.launch.cmd-opts" -> ( "-XX:+UnlockDiagnosticVMOptions -XX:NativeMemoryTracking=summary -XX:+PrintNMTStatistics " + "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp"),and then loading the resulting heap dump into jvisualvm? Following back the most-frequently-used type (beyond string) usually helps to understand what's happening.
--
You received this message because you are subscribed to the Google Groups "cascading-user" group.
To unsubscribe from this group and stop receiving emails from it, send an email to cascading-use...@googlegroups.com.
To post to this group, send email to cascadi...@googlegroups.com.
Visit this group at https://groups.google.com/group/cascading-user.
To view this discussion on the web visit https://groups.google.com/d/msgid/cascading-user/f68909c5-a90d-4558-9338-7a61cea11555%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.