Hi all,
I am evaluating CDAP 3.2.0 on CDH 5.4 using the PageRank Spark example. CDAP is installed using the CSD.
On loading the app into CDAP via the UI and running the Spark program, it fails with the following.
Is this example expected to work out of the box with this mix of versions?
Any help appreciated.
Cheers,
/T
2015-10-30 21:02:56,911 - ERROR [Spark - PageRankSpark:c.c.c.i.a.r.s.SparkRuntimeService@203] - Spark program execution failure: SparkContext{id=spark.default.PageRankApp.PageRankSpark, runId=888fe527-7f49-11e5-8c29-fa163e502f52}
java.util.concurrent.ExecutionException: java.lang.NoClassDefFoundError: org/apache/hadoop/mapred/JobConf
at com.google.common.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:294) ~[com.google.guava.guava-13.0.1.jar:na]
at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:281) ~[com.google.guava.guava-13.0.1.jar:na]
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) ~[com.google.guava.guava-13.0.1.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkRuntimeService.run(SparkRuntimeService.java:181) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:52) [com.google.guava.guava-13.0.1.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkRuntimeService$2$1.run(SparkRuntimeService.java:247) [co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
Caused by: java.lang.NoClassDefFoundError: org/apache/hadoop/mapred/JobConf
at co.cask.cdap.internal.app.runtime.spark.ScalaSparkFacade.<init>(ScalaSparkFacade.java:40) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkProgramWrapper.setupSparkContext(SparkProgramWrapper.java:76) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkProgramWrapper.run(SparkProgramWrapper.java:58) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkProgramWrapper.main(SparkProgramWrapper.java:40) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.7.0_67]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) ~[na:1.7.0_67]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_67]
at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_67]
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:569) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:166) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:189) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:110) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) ~[spark-assembly.jar:na]
at co.cask.cdap.internal.app.runtime.spark.AbstractSparkSubmitter.submit(AbstractSparkSubmitter.java:136) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.AbstractSparkSubmitter$5.run(AbstractSparkSubmitter.java:99) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) ~[na:1.7.0_67]
at java.util.concurrent.FutureTask.run(FutureTask.java:262) ~[na:1.7.0_67]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) ~[na:1.7.0_67]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ~[na:1.7.0_67]
... 1 common frames omitted
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.mapred.JobConf
at java.net.URLClassLoader$1.run(URLClassLoader.java:366) ~[na:1.7.0_67]
at java.net.URLClassLoader$1.run(URLClassLoader.java:355) ~[na:1.7.0_67]
at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67]
at java.net.URLClassLoader.findClass(URLClassLoader.java:354) ~[na:1.7.0_67]
at java.lang.ClassLoader.loadClass(ClassLoader.java:425) ~[na:1.7.0_67]
at java.lang.ClassLoader.loadClass(ClassLoader.java:358) ~[na:1.7.0_67]
... 20 common frames omitted
2015-10-30 21:02:57,137 - ERROR [Spark - PageRankSpark:c.c.c.i.a.r.ProgramControllerServiceAdapter$1@81] - Program terminated with exception
java.util.concurrent.ExecutionException: java.lang.NoClassDefFoundError: org/apache/hadoop/mapred/JobConf
at com.google.common.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:294) ~[com.google.guava.guava-13.0.1.jar:na]
at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:281) ~[com.google.guava.guava-13.0.1.jar:na]
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) ~[com.google.guava.guava-13.0.1.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkRuntimeService.run(SparkRuntimeService.java:181) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:52) ~[com.google.guava.guava-13.0.1.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkRuntimeService$2$1.run(SparkRuntimeService.java:247) [co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
Caused by: java.lang.NoClassDefFoundError: org/apache/hadoop/mapred/JobConf
at co.cask.cdap.internal.app.runtime.spark.ScalaSparkFacade.<init>(ScalaSparkFacade.java:40) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkProgramWrapper.setupSparkContext(SparkProgramWrapper.java:76) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkProgramWrapper.run(SparkProgramWrapper.java:58) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkProgramWrapper.main(SparkProgramWrapper.java:40) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.7.0_67]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) ~[na:1.7.0_67]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_67]
at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_67]
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:569) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:166) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:189) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:110) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) ~[spark-assembly.jar:na]
at co.cask.cdap.internal.app.runtime.spark.AbstractSparkSubmitter.submit(AbstractSparkSubmitter.java:136) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.AbstractSparkSubmitter$5.run(AbstractSparkSubmitter.java:99) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) ~[na:1.7.0_67]
at java.util.concurrent.FutureTask.run(FutureTask.java:262) ~[na:1.7.0_67]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) ~[na:1.7.0_67]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ~[na:1.7.0_67]
... 1 common frames omitted
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.mapred.JobConf
at java.net.URLClassLoader$1.run(URLClassLoader.java:366) ~[na:1.7.0_67]
at java.net.URLClassLoader$1.run(URLClassLoader.java:355) ~[na:1.7.0_67]
at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67]
at java.net.URLClassLoader.findClass(URLClassLoader.java:354) ~[na:1.7.0_67]
at java.lang.ClassLoader.loadClass(ClassLoader.java:425) ~[na:1.7.0_67]
at java.lang.ClassLoader.loadClass(ClassLoader.java:358) ~[na:1.7.0_67]
... 20 common frames omitted
2015-10-30 21:02:57,147 - ERROR [pcontroller-spark.default.PageRankApp.PageRankSpark-888fe527-7f49-11e5-8c29-fa163e502f52:c.c.c.i.a.r.d.AbstractProgramTwillRunnable$1@309] - Program runner error out.
java.util.concurrent.ExecutionException: java.lang.NoClassDefFoundError: org/apache/hadoop/mapred/JobConf
at com.google.common.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:294) ~[com.google.guava.guava-13.0.1.jar:na]
at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:281) ~[com.google.guava.guava-13.0.1.jar:na]
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) ~[com.google.guava.guava-13.0.1.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkRuntimeService.run(SparkRuntimeService.java:181) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:52) ~[com.google.guava.guava-13.0.1.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkRuntimeService$2$1.run(SparkRuntimeService.java:247) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
Caused by: java.lang.NoClassDefFoundError: org/apache/hadoop/mapred/JobConf
at co.cask.cdap.internal.app.runtime.spark.ScalaSparkFacade.<init>(ScalaSparkFacade.java:40) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkProgramWrapper.setupSparkContext(SparkProgramWrapper.java:76) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkProgramWrapper.run(SparkProgramWrapper.java:58) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.SparkProgramWrapper.main(SparkProgramWrapper.java:40) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.7.0_67]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) ~[na:1.7.0_67]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_67]
at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_67]
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:569) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:166) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:189) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:110) ~[spark-assembly.jar:na]
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) ~[spark-assembly.jar:na]
at co.cask.cdap.internal.app.runtime.spark.AbstractSparkSubmitter.submit(AbstractSparkSubmitter.java:136) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at co.cask.cdap.internal.app.runtime.spark.AbstractSparkSubmitter$5.run(AbstractSparkSubmitter.java:99) ~[co.cask.cdap.cdap-app-fabric-3.2.0.jar:na]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) ~[na:1.7.0_67]
at java.util.concurrent.FutureTask.run(FutureTask.java:262) ~[na:1.7.0_67]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_67]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_67]
... 1 common frames omitted
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.mapred.JobConf
at java.net.URLClassLoader$1.run(URLClassLoader.java:366) ~[na:1.7.0_67]
at java.net.URLClassLoader$1.run(URLClassLoader.java:355) ~[na:1.7.0_67]
at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_67]
at java.net.URLClassLoader.findClass(URLClassLoader.java:354) ~[na:1.7.0_67]
at java.lang.ClassLoader.loadClass(ClassLoader.java:425) ~[na:1.7.0_67]
at java.lang.ClassLoader.loadClass(ClassLoader.java:358) ~[na:1.7.0_67]
... 20 common frames omitted