2018-01-19T22:30:52,653 INFO [pool-29-thread-1] io.druid.indexer.HadoopDruidIndexerConfig - Running with config: { "spec" : { "dataSchema" : { "dataSource" : "sauce_jobs_ds", "parser" : { "type" : "hadoopyString", "parseSpec" : { "format" : "json", "dimensionsSpec" : { "dimensions" : [ "id", "creation_time", "assignment_time", "start_time", "end_time", "modification_time", "deletion_time", "owner", "ancestor", "name", "status", "passed", "error", "exception", "tags", "build", "automation_backend", "selenium_version", "browser", "browser_version", "browser_normalized", "os", "os_normalized", "locale", "language", "session_start_time", "session_end_time", "duration_in_progress_sec", "consolidate_status", "reporting_time" ] }, "timestampSpec" : { "format" : "auto", "column" : "reporting_time" } } }, "metricsSpec" : [ { "type" : "count", "name" : "total_jobs" }, { "type" : "doubleSum", "name" : "total_duration_in_progress_sec", "fieldName" : "duration_in_progress_sec", "expression" : null } ], "granularitySpec" : { "type" : "uniform", "segmentGranularity" : "HOUR", "queryGranularity" : { "type" : "none" }, "rollup" : true, "intervals" : [ "2018-01-16T00:00:00.000Z/2018-01-20T00:00:00.000Z" ] } }, "ioConfig" : { "type" : "hadoop", "inputSpec" : { "type" : "static", "paths" : "quickstart/sauce-jobs-data.json" }, "metadataUpdateSpec" : null, "segmentOutputPath" : "s3n://analytics-druid/druid/segments" }, "tuningConfig" : { "type" : "hadoop", "workingPath" : "var/druid/hadoop-tmp", "version" : "2018-01-19T22:30:36.720Z", "partitionsSpec" : { "type" : "hashed", "targetPartitionSize" : 5000000, "maxPartitionSize" : 7500000, "assumeGrouped" : false, "numShards" : -1, "partitionDimensions" : [ ] }, "shardSpecs" : { "1516060800000" : [ ], "1516064400000" : [ ], "1516068000000" : [ ], "1516071600000" : [ ], "1516075200000" : [ ], "1516078800000" : [ ], "1516082400000" : [ ], "1516086000000" : [ ], "1516089600000" : [ ], "1516093200000" : [ ], "1516096800000" : [ ], "1516100400000" : [ ], "1516104000000" : [ ], "1516107600000" : [ ], "1516111200000" : [ ], "1516114800000" : [ ], "1516118400000" : [ ], "1516122000000" : [ ], "1516125600000" : [ ], "1516129200000" : [ { "actualSpec" : { "type" : "none" }, "shardNum" : 0 } ], "1516132800000" : [ ], "1516136400000" : [ ], "1516140000000" : [ ], "1516143600000" : [ ], "1516147200000" : [ ], "1516150800000" : [ ], "1516154400000" : [ ], "1516158000000" : [ ], "1516161600000" : [ ], "1516165200000" : [ ], "1516168800000" : [ ], "1516172400000" : [ ], "1516176000000" : [ ], "1516179600000" : [ ], "1516183200000" : [ ], "1516186800000" : [ ], "1516190400000" : [ ], "1516194000000" : [ ], "1516197600000" : [ { "actualSpec" : { "type" : "none" }, "shardNum" : 1 } ], "1516201200000" : [ ], "1516204800000" : [ ], "1516208400000" : [ ], "1516212000000" : [ ], "1516215600000" : [ ], "1516219200000" : [ ], "1516222800000" : [ ], "1516226400000" : [ ], "1516230000000" : [ ], "1516233600000" : [ ], "1516237200000" : [ ], "1516240800000" : [ ], "1516244400000" : [ ], "1516248000000" : [ ], "1516251600000" : [ ], "1516255200000" : [ ], "1516258800000" : [ ], "1516262400000" : [ ], "1516266000000" : [ ], "1516269600000" : [ ], "1516273200000" : [ ], "1516276800000" : [ ], "1516280400000" : [ ], "1516284000000" : [ ], "1516287600000" : [ ], "1516291200000" : [ ], "1516294800000" : [ ], "1516298400000" : [ ], "1516302000000" : [ ], "1516305600000" : [ ], "1516309200000" : [ ], "1516312800000" : [ ], "1516316400000" : [ ], "1516320000000" : [ ], "1516323600000" : [ ], "1516327200000" : [ ], "1516330800000" : [ ], "1516334400000" : [ ], "1516338000000" : [ ], "1516341600000" : [ ], "1516345200000" : [ ], "1516348800000" : [ ], "1516352400000" : [ ], "1516356000000" : [ ], "1516359600000" : [ ], "1516363200000" : [ ], "1516366800000" : [ ], "1516370400000" : [ ], "1516374000000" : [ ], "1516377600000" : [ ], "1516381200000" : [ ], "1516384800000" : [ ], "1516388400000" : [ ], "1516392000000" : [ ], "1516395600000" : [ ], "1516399200000" : [ ], "1516402800000" : [ ] }, "indexSpec" : { "bitmap" : { "type" : "concise" }, "dimensionCompression" : "lz4", "metricCompression" : "lz4", "longEncoding" : "longs" }, "maxRowsInMemory" : 75000, "leaveIntermediate" : false, "cleanupOnFailure" : true, "overwriteFiles" : false, "ignoreInvalidRows" : false, "jobProperties" : { }, "combineText" : false, "useCombiner" : false, "buildV9Directly" : true, "numBackgroundPersistThreads" : 0, "forceExtendableShardSpecs" : false, "useExplicitVersion" : false, "allowedHadoopPrefix" : [ ] }, "uniqueId" : "73b825a46a6d4564aca42d0405820556" } } 2018-01-19T22:30:52,812 INFO [task-runner-0-priority-0] org.apache.hadoop.mapreduce.Job - map 100% reduce 0% 2018-01-19T22:30:53,051 INFO [pool-29-thread-1] io.druid.indexer.IndexGeneratorJob - 16,040 lines completed. 2018-01-19T22:30:53,051 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - Starting persist for interval[2018-01-17T14:00:00.000Z/2018-01-17T15:00:00.000Z], rows[16,036] 2018-01-19T22:30:53,117 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - Start making v9 index files, outDir:var/tmp/base4488530393210431879flush/merged 2018-01-19T22:30:53,117 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - Completed version.bin in 0 millis. 2018-01-19T22:30:53,117 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - Completed factory.json in 0 millis 2018-01-19T22:30:53,139 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[id] conversions with cardinality[15,372] in 22 millis. 2018-01-19T22:30:53,142 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[creation_time] conversions with cardinality[1,094] in 2 millis. 2018-01-19T22:30:53,145 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[assignment_time] conversions with cardinality[1,048] in 2 millis. 2018-01-19T22:30:53,148 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[start_time] conversions with cardinality[1,053] in 2 millis. 2018-01-19T22:30:53,149 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[end_time] conversions with cardinality[553] in 1 millis. 2018-01-19T22:30:53,151 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[modification_time] conversions with cardinality[388] in 1 millis. 2018-01-19T22:30:53,152 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[deletion_time] conversions with cardinality[143] in 0 millis. 2018-01-19T22:30:53,153 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[owner] conversions with cardinality[401] in 1 millis. 2018-01-19T22:30:53,153 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[ancestor] conversions with cardinality[0] in 0 millis. 2018-01-19T22:30:53,164 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[name] conversions with cardinality[4,620] in 9 millis. 2018-01-19T22:30:53,164 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[status] conversions with cardinality[1] in 0 millis. 2018-01-19T22:30:53,165 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[passed] conversions with cardinality[3] in 0 millis. 2018-01-19T22:30:53,165 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[error] conversions with cardinality[33] in 0 millis. 2018-01-19T22:30:53,166 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[exception] conversions with cardinality[21] in 0 millis. 2018-01-19T22:30:53,167 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[tags] conversions with cardinality[299] in 0 millis. 2018-01-19T22:30:53,169 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[build] conversions with cardinality[619] in 1 millis. 2018-01-19T22:30:53,170 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[automation_backend] conversions with cardinality[2] in 1 millis. 2018-01-19T22:30:53,170 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[selenium_version] conversions with cardinality[22] in 0 millis. 2018-01-19T22:30:53,171 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[browser] conversions with cardinality[8] in 0 millis. 2018-01-19T22:30:53,172 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[browser_version] conversions with cardinality[103] in 1 millis. 2018-01-19T22:30:53,172 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[browser_normalized] conversions with cardinality[191] in 0 millis. 2018-01-19T22:30:53,173 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[os] conversions with cardinality[12] in 0 millis. 2018-01-19T22:30:53,174 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[os_normalized] conversions with cardinality[60] in 1 millis. 2018-01-19T22:30:53,174 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[locale] conversions with cardinality[3] in 0 millis. 2018-01-19T22:30:53,175 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[language] conversions with cardinality[2] in 1 millis. 2018-01-19T22:30:53,177 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[session_start_time] conversions with cardinality[1,064] in 2 millis. 2018-01-19T22:30:53,178 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[session_end_time] conversions with cardinality[619] in 1 millis. 2018-01-19T22:30:53,180 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[duration_in_progress_sec] conversions with cardinality[910] in 2 millis. 2018-01-19T22:30:53,180 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[consolidate_status] conversions with cardinality[4] in 0 millis. 2018-01-19T22:30:53,181 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[reporting_time] conversions with cardinality[388] in 0 millis. 2018-01-19T22:30:53,182 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - Completed dim conversions in 65 millis. 2018-01-19T22:30:53,306 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - completed walk through of 16,036 rows in 124 millis. 2018-01-19T22:30:53,308 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - Completed time column in 2 millis. 2018-01-19T22:30:53,317 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - Completed metric column[total_duration_in_progress_sec] in 9 millis. 2018-01-19T22:30:53,320 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - Completed metric column[total_jobs] in 3 millis. 2018-01-19T22:30:53,320 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - Completed metric columns in 12 millis. 2018-01-19T22:30:53,335 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[id] inverted with cardinality[15,372] in 15 millis. 2018-01-19T22:30:53,342 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[creation_time] inverted with cardinality[1,094] in 2 millis. 2018-01-19T22:30:53,347 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[assignment_time] inverted with cardinality[1,048] in 2 millis. 2018-01-19T22:30:53,354 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[start_time] inverted with cardinality[1,053] in 2 millis. 2018-01-19T22:30:53,361 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[end_time] inverted with cardinality[553] in 2 millis. 2018-01-19T22:30:53,364 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[modification_time] inverted with cardinality[388] in 2 millis. 2018-01-19T22:30:53,366 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[deletion_time] inverted with cardinality[143] in 1 millis. 2018-01-19T22:30:53,376 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[owner] inverted with cardinality[401] in 3 millis. 2018-01-19T22:30:53,386 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[ancestor] inverted with cardinality[0] in 0 millis. 2018-01-19T22:30:53,393 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[name] inverted with cardinality[4,620] in 7 millis. 2018-01-19T22:30:53,397 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[status] inverted with cardinality[1] in 1 millis. 2018-01-19T22:30:53,399 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[passed] inverted with cardinality[3] in 1 millis. 2018-01-19T22:30:53,407 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[error] inverted with cardinality[33] in 1 millis. 2018-01-19T22:30:53,414 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[exception] inverted with cardinality[21] in 1 millis. 2018-01-19T22:30:53,423 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[tags] inverted with cardinality[299] in 1 millis. 2018-01-19T22:30:53,430 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[build] inverted with cardinality[619] in 1 millis. 2018-01-19T22:30:53,436 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[automation_backend] inverted with cardinality[2] in 0 millis. 2018-01-19T22:30:53,441 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[selenium_version] inverted with cardinality[22] in 1 millis. 2018-01-19T22:30:53,449 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[browser] inverted with cardinality[8] in 1 millis. 2018-01-19T22:30:53,456 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[browser_version] inverted with cardinality[103] in 1 millis. 2018-01-19T22:30:53,460 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[browser_normalized] inverted with cardinality[191] in 2 millis. 2018-01-19T22:30:53,464 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[os] inverted with cardinality[12] in 2 millis. 2018-01-19T22:30:53,470 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[os_normalized] inverted with cardinality[60] in 1 millis. 2018-01-19T22:30:53,474 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[locale] inverted with cardinality[3] in 1 millis. 2018-01-19T22:30:53,482 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[language] inverted with cardinality[2] in 1 millis. 2018-01-19T22:30:53,489 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[session_start_time] inverted with cardinality[1,064] in 3 millis. 2018-01-19T22:30:53,497 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[session_end_time] inverted with cardinality[619] in 3 millis. 2018-01-19T22:30:53,501 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[duration_in_progress_sec] inverted with cardinality[910] in 2 millis. 2018-01-19T22:30:53,504 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[consolidate_status] inverted with cardinality[4] in 1 millis. 2018-01-19T22:30:53,511 INFO [pool-29-thread-1] io.druid.segment.StringDimensionMergerV9 - Completed dim[reporting_time] inverted with cardinality[388] in 1 millis. 2018-01-19T22:30:53,513 INFO [pool-29-thread-1] io.druid.segment.IndexMergerV9 - Completed index.drd in 1 millis. 2018-01-19T22:30:53,513 INFO [pool-29-thread-1] io.druid.java.util.common.io.smoosh.FileSmoosher - Created smoosh file [/home/druid/ecosys/druid-0.11.0/var/tmp/base4488530393210431879flush/merged/00000.smoosh] of size [2436087] bytes. 2018-01-19T22:30:53,531 INFO [Thread-931] org.apache.hadoop.mapred.LocalJobRunner - reduce task executor complete. 2018-01-19T22:30:53,539 WARN [Thread-931] org.apache.hadoop.mapred.LocalJobRunner - job_local394638354_0002 java.lang.Exception: java.io.IOException: No FileSystem for scheme: s3n at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462) ~[hadoop-mapreduce-client-common-2.7.3.jar:?] at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529) [hadoop-mapreduce-client-common-2.7.3.jar:?] Caused by: java.io.IOException: No FileSystem for scheme: s3n at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2660) ~[hadoop-common-2.7.3.jar:?] at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2667) ~[hadoop-common-2.7.3.jar:?] at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94) ~[hadoop-common-2.7.3.jar:?] at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2703) ~[hadoop-common-2.7.3.jar:?] at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2685) ~[hadoop-common-2.7.3.jar:?] at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:373) ~[hadoop-common-2.7.3.jar:?] at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295) ~[hadoop-common-2.7.3.jar:?] at io.druid.indexer.IndexGeneratorJob$IndexGeneratorReducer.reduce(IndexGeneratorJob.java:709) ~[druid-indexing-hadoop-0.11.0.jar:0.11.0] at io.druid.indexer.IndexGeneratorJob$IndexGeneratorReducer.reduce(IndexGeneratorJob.java:489) ~[druid-indexing-hadoop-0.11.0.jar:0.11.0] at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) ~[hadoop-mapreduce-client-core-2.7.3.jar:?] at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627) ~[hadoop-mapreduce-client-core-2.7.3.jar:?] at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) ~[hadoop-mapreduce-client-core-2.7.3.jar:?] at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319) ~[hadoop-mapreduce-client-common-2.7.3.jar:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_151] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_151] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_151] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_151] at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_151] 2018-01-19T22:30:53,813 INFO [task-runner-0-priority-0] org.apache.hadoop.mapreduce.Job - Job job_local394638354_0002 failed with state FAILED due to: NA 2018-01-19T22:30:53,825 INFO [task-runner-0-priority-0] org.apache.hadoop.mapreduce.Job - Counters: 30 File System Counters FILE: Number of bytes read=37812016 FILE: Number of bytes written=18453388 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 Map-Reduce Framework Map input records=17372 Map output records=17372 Map output bytes=17780538 Map output materialized bytes=17850038 Input split bytes=298 Combine input records=0 Combine output records=0 Reduce input groups=0 Reduce shuffle bytes=17850038 Reduce input records=0 Reduce output records=0 Spilled Records=17372 Shuffled Maps =2 Failed Shuffles=0 Merged Map outputs=2 GC time elapsed (ms)=59 Total committed heap usage (bytes)=3052404736 Shuffle Errors BAD_ID=0 CONNECTION=0 IO_ERROR=0 WRONG_LENGTH=0 WRONG_MAP=0 WRONG_REDUCE=0 File Input Format Counters Bytes Read=0 File Output Format Counters Bytes Written=0 2018-01-19T22:30:53,831 INFO [task-runner-0-priority-0] io.druid.indexer.JobHelper - Deleting path[var/druid/hadoop-tmp/sauce_jobs_ds/2018-01-19T223036.720Z_73b825a46a6d4564aca42d0405820556] 2018-01-19T22:30:53,858 ERROR [task-runner-0-priority-0] io.druid.indexing.overlord.ThreadPoolTaskRunner - Exception while running task[HadoopIndexTask{id=index_hadoop_sauce_jobs_ds_2018-01-19T22:30:36.715Z, type=index_hadoop, dataSource=sauce_jobs_ds}] java.lang.RuntimeException: java.lang.reflect.InvocationTargetException at com.google.common.base.Throwables.propagate(Throwables.java:160) ~[guava-16.0.1.jar:?] at io.druid.indexing.common.task.HadoopTask.invokeForeignLoader(HadoopTask.java:218) ~[druid-indexing-service-0.11.0.jar:0.11.0] at io.druid.indexing.common.task.HadoopIndexTask.run(HadoopIndexTask.java:226) ~[druid-indexing-service-0.11.0.jar:0.11.0] at io.druid.indexing.overlord.ThreadPoolTaskRunner$ThreadPoolTaskRunnerCallable.call(ThreadPoolTaskRunner.java:436) [druid-indexing-service-0.11.0.jar:0.11.0] at io.druid.indexing.overlord.ThreadPoolTaskRunner$ThreadPoolTaskRunnerCallable.call(ThreadPoolTaskRunner.java:408) [druid-indexing-service-0.11.0.jar:0.11.0] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_151] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_151] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_151] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151] Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_151] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_151] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_151] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_151] at io.druid.indexing.common.task.HadoopTask.invokeForeignLoader(HadoopTask.java:215) ~[druid-indexing-service-0.11.0.jar:0.11.0] ... 7 more Caused by: io.druid.java.util.common.ISE: Job[class io.druid.indexer.IndexGeneratorJob] failed! at io.druid.indexer.JobHelper.runJobs(JobHelper.java:390) ~[druid-indexing-hadoop-0.11.0.jar:0.11.0] at io.druid.indexer.HadoopDruidIndexerJob.run(HadoopDruidIndexerJob.java:95) ~[druid-indexing-hadoop-0.11.0.jar:0.11.0] at io.druid.indexing.common.task.HadoopIndexTask$HadoopIndexGeneratorInnerProcessing.runTask(HadoopIndexTask.java:279) ~[druid-indexing-service-0.11.0.jar:0.11.0] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_151] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_151] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_151] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_151] at io.druid.indexing.common.task.HadoopTask.invokeForeignLoader(HadoopTask.java:215) ~[druid-indexing-service-0.11.0.jar:0.11.0] ... 7 more 2018-01-19T22:30:53,870 INFO [task-runner-0-priority-0] io.druid.indexing.overlord.TaskRunnerUtils - Task [index_hadoop_sauce_jobs_ds_2018-01-19T22:30:36.715Z] status changed to [FAILED]. 2018-01-19T22:30:53,878 INFO [task-runner-0-priority-0] io.druid.indexing.worker.executor.ExecutorLifecycle - Task completed with status: { "id" : "index_hadoop_sauce_jobs_ds_2018-01-19T22:30:36.715Z", "status" : "FAILED", "duration" : 12139 } 2018-01-19T22:30:53,892 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.server.listener.announcer.ListenerResourceAnnouncer.stop()] on object[io.druid.query.lookup.LookupResourceListenerAnnouncer@53ea380b]. 2018-01-19T22:30:53,892 INFO [main] io.druid.curator.announcement.Announcer - unannouncing [/druid/listeners/lookups/__default/http:10.100.48.2:8100] 2018-01-19T22:30:53,907 INFO [main] io.druid.server.listener.announcer.ListenerResourceAnnouncer - Unannouncing start time on [/druid/listeners/lookups/__default/http:10.100.48.2:8100] 2018-01-19T22:30:53,913 INFO [main] org.eclipse.jetty.server.AbstractConnector - Stopped ServerConnector@d25e878{HTTP/1.1,[http/1.1]}{0.0.0.0:8100} 2018-01-19T22:30:53,916 INFO [main] org.eclipse.jetty.server.handler.ContextHandler - Stopped o.e.j.s.ServletContextHandler@226d5af0{/,null,UNAVAILABLE} 2018-01-19T22:30:53,920 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.indexing.worker.executor.ExecutorLifecycle.stop() throws java.lang.Exception] on object[io.druid.indexing.worker.executor.ExecutorLifecycle@5707f613]. 2018-01-19T22:30:53,921 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.indexing.overlord.ThreadPoolTaskRunner.stop()] on object[io.druid.indexing.overlord.ThreadPoolTaskRunner@79e10fb4]. 2018-01-19T22:30:53,921 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.curator.announcement.Announcer.stop()] on object[io.druid.curator.announcement.Announcer@764b14b8]. 2018-01-19T22:30:53,922 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.discovery.DruidLeaderClient.stop()] on object[io.druid.discovery.DruidLeaderClient@721d5b74]. 2018-01-19T22:30:53,922 INFO [main] io.druid.discovery.DruidLeaderClient - Stopped. 2018-01-19T22:30:53,922 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.curator.discovery.ServerDiscoverySelector.stop() throws java.io.IOException] on object[io.druid.curator.discovery.ServerDiscoverySelector@7a2fce12]. 2018-01-19T22:30:53,924 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void com.metamx.metrics.MonitorScheduler.stop()] on object[com.metamx.metrics.MonitorScheduler@310b2b6f]. 2018-01-19T22:30:53,924 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.query.lookup.LookupReferencesManager.stop()] on object[io.druid.query.lookup.LookupReferencesManager@7e87ef9e]. 2018-01-19T22:30:53,925 INFO [main] io.druid.query.lookup.LookupReferencesManager - LookupReferencesManager is stopping. 2018-01-19T22:30:53,925 INFO [LookupReferencesManager-MainThread] io.druid.query.lookup.LookupReferencesManager - Lookup Management loop exited, Lookup notices are not handled anymore. 2018-01-19T22:30:53,925 INFO [main] io.druid.query.lookup.LookupReferencesManager - LookupReferencesManager is stopped. 2018-01-19T22:30:53,925 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.discovery.DruidLeaderClient.stop()] on object[io.druid.discovery.DruidLeaderClient@64aeaf29]. 2018-01-19T22:30:53,925 INFO [main] io.druid.discovery.DruidLeaderClient - Stopped. 2018-01-19T22:30:53,925 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.curator.discovery.ServerDiscoverySelector.stop() throws java.io.IOException] on object[io.druid.curator.discovery.ServerDiscoverySelector@abad89c]. 2018-01-19T22:30:53,926 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.curator.discovery.CuratorDruidNodeDiscoveryProvider.stop()] on object[io.druid.curator.discovery.CuratorDruidNodeDiscoveryProvider@6eb089e6]. 2018-01-19T22:30:53,926 INFO [main] io.druid.curator.discovery.CuratorDruidNodeDiscoveryProvider - stopping 2018-01-19T22:30:53,926 INFO [main] io.druid.curator.discovery.CuratorDruidNodeDiscoveryProvider - stopped 2018-01-19T22:30:53,927 INFO [main] com.metamx.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void com.metamx.http.client.NettyHttpClient.stop()] on object[com.metamx.http.client.NettyHttpClient@5d7d8613]. 2018-01-19T22:30:53,992 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void com.metamx.emitter.service.ServiceEmitter.close() throws java.io.IOException] on object[ServiceEmitter{serviceDimensions={service=druid/middleManager, host=10.100.48.2:8100, version=0.11.0}, emitter=LoggingEmitter{log=com.metamx.common.logger.Logger@44a6a68e, level=INFO}}]. 2018-01-19T22:30:53,992 INFO [main] com.metamx.emitter.core.LoggingEmitter - Close: started [false] 2018-01-19T22:30:53,992 INFO [main] io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking stop method[public void io.druid.initialization.Log4jShutterDownerModule$Log4jShutterDowner.stop()] on object[io.druid.initialization.Log4jShutterDownerModule$Log4jShutterDowner@79980d8d].