Exception in thread "main" java.lang.NoSuchMethodException: org.apache.hadoop.mapreduce.Job.getJobClient()
at java.lang.Class.getDeclaredMethod(Class.java:1937)
at com.nicta.scoobi.impl.reflect.Classes$class.invokeProtected(Classes.scala:135)
at com.nicta.scoobi.impl.reflect.Classes$.invokeProtected(Classes.scala:165)
at com.nicta.scoobi.impl.exec.TaskDetailsLogger.getJobClient$lzycompute(MapReduceJob.scala:314)
at com.nicta.scoobi.impl.exec.TaskDetailsLogger.getJobClient(MapReduceJob.scala:314)
at com.nicta.scoobi.impl.exec.TaskDetailsLogger.com$nicta$scoobi$impl$exec$TaskDetailsLogger$$getTaskCompletionEvents(MapReduceJob.scala:308)
at com.nicta.scoobi.impl.exec.TaskDetailsLogger$$anonfun$logTaskCompletionDetails$1.apply(MapReduceJob.scala:285)
at com.nicta.scoobi.impl.exec.TaskDetailsLogger$$anonfun$logTaskCompletionDetails$1.apply(MapReduceJob.scala:285)
I am looking into a workaround and will keep you posted, but I am open to suggestions.
I also have a patch to bring the scoobi build to the latests version of hadoop2, but it will not work without fixing the getJobClient problem.
--- a/project/dependencies.scala
+++ b/project/dependencies.scala
@@ -38,13 +38,13 @@ object dependencies {
"org.apache.commons" % "commons-compress" % "1.0" % "test")
def hadoop(version: String) =
- if (version.contains("hadoop2")) Seq("org.apache.hadoop" % "hadoop-common" % "2.1.0.2.0.5.0-67",
- "org.apache.hadoop" % "hadoop-hdfs" % "2.1.0.2.0.5.0-67",
- "org.apache.hadoop" % "hadoop-mapreduce-client-app" % "2.1.0.2.0.5.0-67",
- "org.apache.hadoop" % "hadoop-mapreduce-client-core" % "2.1.0.2.0.5.0-67",
- "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % "2.1.0.2.0.5.0-67",
- "org.apache.hadoop" % "hadoop-mapreduce-client-core" % "2.1.0.2.0.5.0-67",
- "org.apache.hadoop" % "hadoop-annotations" % "2.1.0.2.0.5.0-67",
+ if (version.contains("hadoop2")) Seq("org.apache.hadoop" % "hadoop-common" % "2.2.0.2.0.6.0-76",
+ "org.apache.hadoop" % "hadoop-hdfs" % "2.2.0.2.0.6.0-76",
+ "org.apache.hadoop" % "hadoop-mapreduce-client-app" % "2.2.0.2.0.6.0-76",
+ "org.apache.hadoop" % "hadoop-mapreduce-client-core" % "2.2.0.2.0.6.0-76",
+ "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % "2.2.0.2.0.6.0-76",
+ "org.apache.hadoop" % "hadoop-mapreduce-client-core" % "2.2.0.2.0.6.0-76",
+ "org.apache.hadoop" % "hadoop-annotations" % "2.2.0.2.0.6.0-76",
"org.apache.avro" % "avro-mapred" % "1.7.4")
else if (version.contains("cdh3")) Seq("org.apache.hadoop" % "hadoop-core" % "0.20.2-cdh3u1",
"org.apache.avro" % "avro-mapred" % "1.7.4")