# Copyright 2019 JanusGraph Authors## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.# You may obtain a copy of the License at### Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.
## Hadoop Graph Configuration#gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraphgremlin.hadoop.graphReader=org.janusgraph.hadoop.formats.cql.CqlInputFormatgremlin.hadoop.graphWriter=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormatgremlin.hadoop.defaultGraphComputer=org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer
gremlin.hadoop.jarsInDistributedCache=truegremlin.hadoop.inputLocation=nonegremlin.hadoop.outputLocation=outputgremlin.spark.persistContext=true## JanusGraph Cassandra InputFormat configuration#janusgraphmr.ioformat.conf.storage.backend=cqljanusgraphmr.ioformat.conf.storage.hostname=X.0.X.1janusgraphmr.ioformat.conf.storage.port=9042janusgraphmr.ioformat.conf.storage.cassandra.keyspace=graph1storage.cassandra.keyspace=graph1
## Apache Cassandra InputFormat configuration#cassandra.input.partitioner.class=org.apache.cassandra.dht.Murmur3Partitioner
## SparkGraphComputer Configuration##spark.master=spark://X.X.X.X:7077spark.master=yarnspark.submit.deployMode=clientspark.yarn.jars=/usr/lib/spark/jars/spark.serializer=org.apache.spark.serializer.KryoSerializerspark.kryo.registrator=org.janusgraph.hadoop.serialize.JanusGraphKryoRegistrator
ava.lang.IllegalStateException: org.apache.spark.SparkException: Unable to load YARN support at org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep.processNextStart(VertexProgramStep.java:88) at org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep.hasNext(AbstractStep.java:143) at org.apache.tinkerpop.gremlin.process.traversal.step.util.ExpandableStepIterator.next(ExpandableStepIterator.java:50) at org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.ComputerResultStep.processNextStart(ComputerResultStep.java:68) at org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep.hasNext(AbstractStep.java:143) at org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal.hasNext(DefaultTraversal.java:192) at org.codehaus.groovy.vmplugin.v7.IndyInterface.selectMethod(IndyInterface.java:236) at org.apache.tinkerpop.gremlin.console.Console$_closure3.doCall(Console.groovy:214) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:264) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1034) at org.codehaus.groovy.tools.shell.Groovysh.setLastResult(Groovysh.groovy:460) at org.codehaus.groovy.vmplugin.v7.IndyInterface.selectMethod(IndyInterface.java:236) at org.codehaus.groovy.tools.shell.Groovysh.execute(Groovysh.groovy:196) at org.apache.tinkerpop.gremlin.console.GremlinGroovysh.super$3$execute(GremlinGroovysh.groovy) at sun.reflect.GeneratedMethodAccessor14.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1225) at org.codehaus.groovy.runtime.ScriptBytecodeAdapter.invokeMethodOnSuperN(ScriptBytecodeAdapter.java:145) at org.apache.tinkerpop.gremlin.console.GremlinGroovysh.execute(GremlinGroovysh.groovy:72) at org.codehaus.groovy.tools.shell.Shell.leftShift(Shell.groovy:122) at org.codehaus.groovy.tools.shell.ShellRunner.work(ShellRunner.groovy:95) at org.codehaus.groovy.tools.shell.InteractiveShellRunner.super$2$work(InteractiveShellRunner.groovy) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1225) at org.codehaus.groovy.runtime.ScriptBytecodeAdapter.invokeMethodOnSuperN(ScriptBytecodeAdapter.java:145) at org.codehaus.groovy.runtime.ScriptBytecodeAdapter.invokeMethodOnSuper0(ScriptBytecodeAdapter.java:165) at org.codehaus.groovy.tools.shell.InteractiveShellRunner.work(InteractiveShellRunner.groovy:130) at org.codehaus.groovy.tools.shell.ShellRunner.run(ShellRunner.groovy:59) at org.codehaus.groovy.tools.shell.InteractiveShellRunner.super$2$run(InteractiveShellRunner.groovy) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1225) at org.codehaus.groovy.runtime.ScriptBytecodeAdapter.invokeMethodOnSuperN(ScriptBytecodeAdapter.java:145) at org.codehaus.groovy.runtime.ScriptBytecodeAdapter.invokeMethodOnSuper0(ScriptBytecodeAdapter.java:165) at org.codehaus.groovy.tools.shell.InteractiveShellRunner.run(InteractiveShellRunner.groovy:89) at org.codehaus.groovy.vmplugin.v7.IndyInterface.selectMethod(IndyInterface.java:236) at org.apache.tinkerpop.gremlin.console.Console.<init>(Console.groovy:146) at org.codehaus.groovy.vmplugin.v7.IndyInterface.selectMethod(IndyInterface.java:236) at org.apache.tinkerpop.gremlin.console.Console.main(Console.groovy:453)Caused by: java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Unable to load YARN support at java.util.concurrent.FutureTask.report(FutureTask.java:122) at java.util.concurrent.FutureTask.get(FutureTask.java:192) at org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep.processNextStart(VertexProgramStep.java:68) ... 56 moreCaused by: org.apache.spark.SparkException: Unable to load YARN support at org.apache.spark.deploy.SparkHadoopUtil$.liftedTree1$1(SparkHadoopUtil.scala:405) at org.apache.spark.deploy.SparkHadoopUtil$.yarn$lzycompute(SparkHadoopUtil.scala:400) at org.apache.spark.deploy.SparkHadoopUtil$.yarn(SparkHadoopUtil.scala:400) at org.apache.spark.deploy.SparkHadoopUtil$.get(SparkHadoopUtil.scala:425) at org.apache.spark.util.Utils$.getSparkOrYarnConfig(Utils.scala:2387) at org.apache.spark.storage.BlockManager.<init>(BlockManager.scala:156) at org.apache.spark.SparkEnv$.create(SparkEnv.scala:351) at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:175) at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:257) at org.apache.spark.SparkContext.<init>(SparkContext.scala:432) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2509) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.tinkerpop.gremlin.spark.structure.Spark.create(Spark.java:52) at org.apache.tinkerpop.gremlin.spark.structure.Spark.create(Spark.java:60) at org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer.lambda$submitWithExecutor$1(SparkGraphComputer.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)Caused by: java.lang.ClassNotFoundException: org.apache.spark.deploy.yarn.YarnSparkHadoopUtil at java.net.URLClassLoader.findClass(URLClassLoader.java:382) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.apache.spark.util.Utils$.classForName(Utils.scala:230) at org.apache.spark.deploy.SparkHadoopUtil$.liftedTree1$1(SparkHadoopUtil.scala:401) ... 18 more
gremlin> graph = GraphFactory.open('conf/hadoop-graph/read-cql-syclla.properties')==>hadoopgraph[cqlinputformat->gryooutputformat]gremlin> g = graph.traversal().withComputer(SparkGraphComputer)==>graphtraversalsource[hadoopgraph[cqlinputformat->gryooutputformat], sparkgraphcomputer]gremlin> g.V().limit(2)java.lang.NullPointerExceptionType ':help' or ':h' for help.Display stack trace? [yN]yjava.lang.IllegalStateException: java.lang.NullPointerException
Caused by: java.util.concurrent.ExecutionException: java.lang.NullPointerException
at java.util.concurrent.FutureTask.report(FutureTask.java:122) at java.util.concurrent.FutureTask.get(FutureTask.java:192) at org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep.processNextStart(VertexProgramStep.java:68) ... 56 more
Caused by: java.lang.NullPointerException at org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer.loadJars(AbstractHadoopGraphComputer.java:169) at org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer.lambda$submitWithExecutor$1(SparkGraphComputer.java:251)
at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.
## Hadoop Graph Configuration#gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraphgremlin.hadoop.graphReader=org.janusgraph.hadoop.formats.cql.CqlInputFormatgremlin.hadoop.graphWriter=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat
gremlin.hadoop.jarsInDistributedCache=truegremlin.hadoop.inputLocation=nonegremlin.hadoop.outputLocation=output
## JanusGraph Cassandra InputFormat configuration#janusgraphmr.ioformat.conf.storage.backend=cql
janusgraphmr.ioformat.conf.storage.hostname=10.X.X.Xjanusgraphmr.ioformat.conf.storage.port=9042janusgraphmr.ioformat.conf.storage.cassandra.keyspace=graphname
## Apache Cassandra InputFormat configuration#cassandra.input.partitioner.class=org.apache.cassandra.dht.Murmur3Partitioner
cassandra.input.keyspace= graphnamecassandra.input.predicate=0c00020b0001000000000b000200000000020003000800047fffffff0000cassandra.input.columnfamily=edgestorecassandra.range.batch.size=214748364
## SparkGraphComputer Configuration#
spark.master=yarnspark.deploy.mode=client#spark.master=local[4]spark.serializer=org.apache.spark.serializer.KryoSerializerspark.kryo.registrator=org.janusgraph.hadoop.serialize.JanusGraphKryoRegistrator
gremlin.spark.persistContext=true
# Default Graph Computergremlin.hadoop.defaultGraphComputer=org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer
Hello Nitin,
Looks like your installation lacks required Spark jars -
https://groups.google.com/d/msg/gremlin-users/LYv-cvZ66hU/TJUTvLzCAAAJ
- you have to provide full installation
Please, also see:
Best regards,
Evgenii Ignatev.
--
You received this message because you are subscribed to the Google Groups "JanusGraph users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to janusgraph-use...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/janusgraph-users/b65d7bd7-2df4-42c3-819e-f6bd15c9825a%40googlegroups.com.
-- Best regards, Evgeniy Ignatiev.
#!/bin/bash
GREMLIN_HOME=/tmp/janusgraph-0.4.0-hadoop2cd $GREMLIN_HOME
# Have janusgraph find the hadoop and hbase cluster configs and spark-yarn dependencies#export CLASSPATH=/etc/hadoop/conf:$GREMLIN_HOME/lib/*:$GREMLIN_HOME/lib2/*:/usr/lib/hadoop/client/*:/usr/lib/hadoop/*:/usr/lib/spark/jars/*
export SPARK_HOME=/usr/lib/spark
export CLASSPATH=/etc/hadoop/conf:/usr/lib/spark/jars/*:$GREMLIN_HOME/lib/*:$GREMLIN_HOME/lib2/*:/usr/lib/hadoop/client/*:/usr/lib/hadoop/*
# Have hadoop find its native libraries# export JAVA_OPTIONS="-Djava.library.path=/usr/lib/hadoop/client/*:/usr/lib/hadoop/*:/usr/lib/spark/jars/*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop-yarn/*"export JAVA_OPTIONS="-Djava.library.path=/usr/lib/spark/jars/*:/usr/lib/hadoop/client/*:/usr/lib/hadoop/*"
# Does not work for spark-yarn, see spark.yarn.appMasterEnv.CLASSPATH and# spark.executor.extraClassPath. Set nevertheless to get rid of the warning.export HADOOP_GREMLIN_LIBS=$GREMLIN_HOME/empty
bin/gremlin.sh
To unsubscribe from this group and stop receiving emails from it, send an email to janusgra...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/janusgraph-users/b65d7bd7-2df4-42c3-819e-f6bd15c9825a%40googlegroups.com.
To unsubscribe from this group and stop receiving emails from it, send an email to janusgra...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/janusgraph-users/b65d7bd7-2df4-42c3-819e-f6bd15c9825a%40googlegroups.com.
# Copyright 2019 JanusGraph Authors## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.# You may obtain a copy of the License at### Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.
## Hadoop Graph Configuration#gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraphgremlin.hadoop.graphReader=org.janusgraph.hadoop.formats.cql.CqlInputFormatgremlin.hadoop.graphWriter=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat
gremlin.hadoop.jarsInDistributedCache=truegremlin.hadoop.inputLocation=none# gremlin.hadoop.outputLocation=output# gremlin.hadoop.outputLocation=s3://bucket-name/jgdump_20190711gremlin.hadoop.outputLocation=/tmp/jgdump_20190711_2
## JanusGraph Cassandra InputFormat configuration#janusgraphmr.ioformat.conf.storage.backend=cql
janusgraphmr.ioformat.conf.storage.hostname=<hostName>janusgraphmr.ioformat.conf.storage.port=9042janusgraphmr.ioformat.conf.storage.cql.keyspace=keyspaceName
## Apache Cassandra InputFormat configuration#cassandra.input.partitioner.class=org.apache.cassandra.dht.Murmur3Partitioner
cassandra.input.widerows=falsecassandra.range.batch.size=2147483640
cql.input.partitioner.class=org.apache.cassandra.dht.Murmur3Partitionercql.input.widerows=falsecql.range.batch.size=2147483640
spark.master=yarnspark.deploy.mode=clientspark.jars=hdfs:///tmp/janusgraph-0.4.0-hadoop2/lib/spark.yarn.queue=long_runspark.serializer=org.apache.spark.serializer.KryoSerializer#gremlin.spark.graphStorageLevel=MEMORY_AND_DISKgremlin.spark.graphStorageLevel=DISK_ONLYspark.kryo.registrator=org.janusgraph.hadoop.serialize.JanusGraphKryoRegistratorspark.executor.extraJavaOptions="-verbose:class"spark.executor.instances=500spark.executor.cores=2spark.executor.memory=9gspark.yarn.executor.memoryOverhead=2gspark.driver.memory=5gspark.dynamicAllocation.enabled=truespark.shuffle.service.enabled=true
gremlin.spark.persistContext=true
# Default Graph Computergremlin.hadoop.defaultGraphComputer=org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer
spark.cassandra.input.page.row.size=1000000spark.cassandra.input.split.size=1000000spark.cassandra.input.split.size_in_mb=256
spark.cql.input.page.row.size=1000000spark.cql.input.split.size=1000000spark.cql.input.split.size_in_mb=256
mapred.max.split.size=268435456mapreduce.input.fileinputformat.split=268435456spark.network.timeout=240
How does ES affects OLAP performance? Correct me, if I am wrong,
but unless it is explicitly used in Spark custom code, JanusGraph
integration will not leverage it, and it is definitely not being
contacted when loading graph data in-memory for Spark
VertexProgram execution.
Best regards,
Evgenii Ignatev.
To unsubscribe from this group and stop receiving emails from it, send an email to janusgraph-use...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/janusgraph-users/b4d5f6a0-2c44-484a-b02a-4628ea81dc66%40googlegroups.com.