Batch index task failed with java.lang.OutOfMemoryError: Direct buffer memory, version 0.9.1.1 and MM config:
# Task launch parameters
druid.indexer.runner.javaCommand=/usr/lib/jvm/java-7-openjdk-amd64/bin/java
druid.indexer.runner.javaOpts=-server -Xmx1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
druid.indexer.task.baseTaskDir=/srv/nbs/0/druid/task/
# HTTP server threads
druid.server.http.numThreads=25
# Processing threads and buffers
druid.processing.buffer.sizeBytes=256870912
druid.processing.numThreads=2
The exception log:
2016-08-28T18:16:48,145 ERROR [task-runner-0-priority-0] io.druid.indexing.overlord.ThreadPoolTaskRunner - Uncaught Throwable while running task[IndexTask{id=index_test_datasource_2016-08-28T17:19:00.934Z, type=index, dataSource=test_datasource}]
java.lang.OutOfMemoryError: Direct buffer memory
at java.nio.Bits.reserveMemory(Bits.java:631) ~[?:1.7.0_111]
at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123) ~[?:1.7.0_111]
at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:306) ~[?:1.7.0_111]
at io.druid.segment.CompressedPools$4.get(CompressedPools.java:100) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.CompressedPools$4.get(CompressedPools.java:93) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.collections.StupidPool.take(StupidPool.java:64) ~[druid-common-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.CompressedPools.getByteBuf(CompressedPools.java:108) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.CompressedObjectStrategy.fromByteBuffer(CompressedObjectStrategy.java:286) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.CompressedObjectStrategy.fromByteBuffer(CompressedObjectStrategy.java:42) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.GenericIndexed$BufferIndexed._get(GenericIndexed.java:225) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.GenericIndexed$1.get(GenericIndexed.java:300) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.CompressedVSizeIntsIndexedSupplier$CompressedVSizeIndexedInts.loadBuffer(CompressedVSizeIntsIndexedSupplier.java:383) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.CompressedVSizeIntsIndexedSupplier$CompressedVSizeIndexedInts.get(CompressedVSizeIntsIndexedSupplier.java:344) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.column.SimpleDictionaryEncodedColumn.getSingleValueRow(SimpleDictionaryEncodedColumn.java:65) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.QueryableIndexIndexableAdapter$2$1.next(QueryableIndexIndexableAdapter.java:257) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.QueryableIndexIndexableAdapter$2$1.next(QueryableIndexIndexableAdapter.java:177) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at com.google.common.collect.TransformedIterator.next(TransformedIterator.java:48) ~[guava-16.0.1.jar:?]
at com.google.common.collect.Iterators$PeekingImpl.peek(Iterators.java:1162) ~[guava-16.0.1.jar:?]
at com.metamx.common.guava.MergeIterator$1.compare(MergeIterator.java:46) ~[java-util-0.27.9.jar:?]
at com.metamx.common.guava.MergeIterator$1.compare(MergeIterator.java:42) ~[java-util-0.27.9.jar:?]
at java.util.PriorityQueue.siftUpUsingComparator(PriorityQueue.java:649) ~[?:1.7.0_111]
at java.util.PriorityQueue.siftUp(PriorityQueue.java:627) ~[?:1.7.0_111]
at java.util.PriorityQueue.offer(PriorityQueue.java:329) ~[?:1.7.0_111]
at java.util.PriorityQueue.add(PriorityQueue.java:306) ~[?:1.7.0_111]
at com.metamx.common.guava.MergeIterator.<init>(MergeIterator.java:55) ~[java-util-0.27.9.jar:?]
at com.metamx.common.guava.MergeIterable.iterator(MergeIterable.java:49) ~[java-util-0.27.9.jar:?]
at io.druid.collections.CombiningIterable.iterator(CombiningIterable.java:95) ~[druid-common-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.IndexMergerV9.mergeIndexesAndWriteColumns(IndexMergerV9.java:680) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.IndexMergerV9.makeIndexFiles(IndexMergerV9.java:222) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.IndexMerger.merge(IndexMerger.java:423) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.IndexMerger.mergeQueryableIndex(IndexMerger.java:244) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.IndexMerger.mergeQueryableIndex(IndexMerger.java:217) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.indexing.common.index.YeOldePlumberSchool$1.finishJob(YeOldePlumberSchool.java:191) ~[druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at io.druid.indexing.common.task.IndexTask.generateSegment(IndexTask.java:415) ~[druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at io.druid.indexing.common.task.IndexTask.run(IndexTask.java:221) ~[druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at io.druid.indexing.overlord.ThreadPoolTaskRunner$ThreadPoolTaskRunnerCallable.call(ThreadPoolTaskRunner.java:436) [druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at io.druid.indexing.overlord.ThreadPoolTaskRunner$ThreadPoolTaskRunnerCallable.call(ThreadPoolTaskRunner.java:408) [druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at java.util.concurrent.FutureTask.run(FutureTask.java:262) [?:1.7.0_111]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [?:1.7.0_111]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [?:1.7.0_111]
at java.lang.Thread.run(Thread.java:745) [?:1.7.0_111]
2016-08-28T18:16:48,152 ERROR [main] io.druid.cli.CliPeon - Error when starting up. Failing.
java.lang.RuntimeException: java.util.concurrent.ExecutionException: java.lang.OutOfMemoryError: Direct buffer memory
at com.google.common.base.Throwables.propagate(Throwables.java:160) ~[guava-16.0.1.jar:?]
at io.druid.indexing.worker.executor.ExecutorLifecycle.join(ExecutorLifecycle.java:211) ~[druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at io.druid.cli.CliPeon.run(CliPeon.java:287) [druid-services-0.9.1.1.jar:0.9.1.1]
at io.druid.cli.Main.main(Main.java:105) [druid-services-0.9.1.1.jar:0.9.1.1]
Caused by: java.util.concurrent.ExecutionException: java.lang.OutOfMemoryError: Direct buffer memory
at com.google.common.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:299) ~[guava-16.0.1.jar:?]
at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:286) ~[guava-16.0.1.jar:?]
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) ~[guava-16.0.1.jar:?]
at io.druid.indexing.worker.executor.ExecutorLifecycle.join(ExecutorLifecycle.java:208) ~[druid-indexing-service-0.9.1.1.jar:0.9.1.1]
... 2 more
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
at java.nio.Bits.reserveMemory(Bits.java:631) ~[?:1.7.0_111]
at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123) ~[?:1.7.0_111]
at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:306) ~[?:1.7.0_111]
at io.druid.segment.CompressedPools$4.get(CompressedPools.java:100) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.CompressedPools$4.get(CompressedPools.java:93) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.collections.StupidPool.take(StupidPool.java:64) ~[druid-common-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.CompressedPools.getByteBuf(CompressedPools.java:108) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.CompressedObjectStrategy.fromByteBuffer(CompressedObjectStrategy.java:286) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.CompressedObjectStrategy.fromByteBuffer(CompressedObjectStrategy.java:42) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.GenericIndexed$BufferIndexed._get(GenericIndexed.java:225) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.GenericIndexed$1.get(GenericIndexed.java:300) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.CompressedVSizeIntsIndexedSupplier$CompressedVSizeIndexedInts.loadBuffer(CompressedVSizeIntsIndexedSupplier.java:383) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.data.CompressedVSizeIntsIndexedSupplier$CompressedVSizeIndexedInts.get(CompressedVSizeIntsIndexedSupplier.java:344) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.column.SimpleDictionaryEncodedColumn.getSingleValueRow(SimpleDictionaryEncodedColumn.java:65) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.QueryableIndexIndexableAdapter$2$1.next(QueryableIndexIndexableAdapter.java:257) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.QueryableIndexIndexableAdapter$2$1.next(QueryableIndexIndexableAdapter.java:177) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at com.google.common.collect.TransformedIterator.next(TransformedIterator.java:48) ~[guava-16.0.1.jar:?]
at com.google.common.collect.Iterators$PeekingImpl.peek(Iterators.java:1162) ~[guava-16.0.1.jar:?]
at com.metamx.common.guava.MergeIterator$1.compare(MergeIterator.java:46) ~[java-util-0.27.9.jar:?]
at com.metamx.common.guava.MergeIterator$1.compare(MergeIterator.java:42) ~[java-util-0.27.9.jar:?]
at java.util.PriorityQueue.siftUpUsingComparator(PriorityQueue.java:649) ~[?:1.7.0_111]
at java.util.PriorityQueue.siftUp(PriorityQueue.java:627) ~[?:1.7.0_111]
at java.util.PriorityQueue.offer(PriorityQueue.java:329) ~[?:1.7.0_111]
at java.util.PriorityQueue.add(PriorityQueue.java:306) ~[?:1.7.0_111]
at com.metamx.common.guava.MergeIterator.<init>(MergeIterator.java:55) ~[java-util-0.27.9.jar:?]
at com.metamx.common.guava.MergeIterable.iterator(MergeIterable.java:49) ~[java-util-0.27.9.jar:?]
at io.druid.collections.CombiningIterable.iterator(CombiningIterable.java:95) ~[druid-common-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.IndexMergerV9.mergeIndexesAndWriteColumns(IndexMergerV9.java:680) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.IndexMergerV9.makeIndexFiles(IndexMergerV9.java:222) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.IndexMerger.merge(IndexMerger.java:423) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.IndexMerger.mergeQueryableIndex(IndexMerger.java:244) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.segment.IndexMerger.mergeQueryableIndex(IndexMerger.java:217) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.indexing.common.index.YeOldePlumberSchool$1.finishJob(YeOldePlumberSchool.java:191) ~[druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at io.druid.indexing.common.task.IndexTask.generateSegment(IndexTask.java:415) ~[druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at io.druid.indexing.common.task.IndexTask.run(IndexTask.java:221) ~[druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at io.druid.indexing.overlord.ThreadPoolTaskRunner$ThreadPoolTaskRunnerCallable.call(ThreadPoolTaskRunner.java:436) ~[druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at io.druid.indexing.overlord.ThreadPoolTaskRunner$ThreadPoolTaskRunnerCallable.call(ThreadPoolTaskRunner.java:408) ~[druid-indexing-service-0.9.1.1.jar:0.9.1.1]
at java.util.concurrent.FutureTask.run(FutureTask.java:262) ~[?:1.7.0_111]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) ~[?:1.7.0_111]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ~[?:1.7.0_111]
at java.lang.Thread.run(Thread.java:745) ~[?:1.7.0_111]