query error

hexi...@gmail.com

unread,

Jul 31, 2015, 3:24:02 AM7/31/15

to Druid User

io.druid.query.GroupByParallelQueryRunner - Exception with one of the sequences!

java.lang.IndexOutOfBoundsException

at java.nio.Buffer.checkIndex(Buffer.java:546) ~[?:1.8.0_45]

at java.nio.DirectByteBuffer.getInt(DirectByteBuffer.java:681) ~[?:1.8.0_45]

at io.druid.segment.data.VSizeIndexedInts.get(VSizeIndexedInts.java:119) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.segment.column.SimpleDictionaryEncodedColumn.getSingleValueRow(SimpleDictionaryEncodedColumn.java:61) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.segment.QueryableIndexStorageAdapter$CursorSequenceBuilder$1$1$2$1.iterator(QueryableIndexStorageAdapter.java:345) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.groupby.GroupByQueryEngine$RowUpdater.updateValues(GroupByQueryEngine.java:189) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.groupby.GroupByQueryEngine$RowUpdater.access$100(GroupByQueryEngine.java:142) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.groupby.GroupByQueryEngine$RowIterator.next(GroupByQueryEngine.java:367) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.groupby.GroupByQueryEngine$RowIterator.next(GroupByQueryEngine.java:285) ~[druid-processing-0.7.0.jar:0.7.0]

at com.metamx.common.guava.BaseSequence.makeYielder(BaseSequence.java:104) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:81) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.BaseSequence.accumulate(BaseSequence.java:67) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.ConcatSequence$1.accumulate(ConcatSequence.java:46) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.ConcatSequence$1.accumulate(ConcatSequence.java:42) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.YieldingAccumulators$1.accumulate(YieldingAccumulators.java:32) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.MappingYieldingAccumulator.accumulate(MappingYieldingAccumulator.java:57) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.FilteringYieldingAccumulator.accumulate(FilteringYieldingAccumulator.java:69) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.MappingYieldingAccumulator.accumulate(MappingYieldingAccumulator.java:57) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.BaseSequence.makeYielder(BaseSequence.java:104) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:81) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.MappedSequence.toYielder(MappedSequence.java:46) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.ResourceClosingSequence.toYielder(ResourceClosingSequence.java:41) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.FilteredSequence.toYielder(FilteredSequence.java:52) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.MappedSequence.toYielder(MappedSequence.java:46) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.ResourceClosingSequence.toYielder(ResourceClosingSequence.java:41) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.YieldingSequenceBase.accumulate(YieldingSequenceBase.java:34) ~[java-util-0.26.14.jar:?]

at com.metamx.common.guava.ConcatSequence.accumulate(ConcatSequence.java:40) ~[java-util-0.26.14.jar:?]

at io.druid.query.GroupByParallelQueryRunner$1$1.call(GroupByParallelQueryRunner.java:116) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.GroupByParallelQueryRunner$1$1.call(GroupByParallelQueryRunner.java:106) ~[druid-processing-0.7.0.jar:0.7.0]

at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_45]

at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) ~[guava-16.0.1.jar:?]

at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134) ~[?:1.8.0_45]

at com.google.common.util.concurrent.AbstractListeningExecutorService.submit(AbstractListeningExecutorService.java:58) ~[guava-16.0.1.jar:?]

at io.druid.query.GroupByParallelQueryRunner$1.apply(GroupByParallelQueryRunner.java:104) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.GroupByParallelQueryRunner$1.apply(GroupByParallelQueryRunner.java:96) ~[druid-processing-0.7.0.jar:0.7.0]

at com.google.common.collect.Iterators$8.transform(Iterators.java:794) ~[guava-16.0.1.jar:?]

at com.google.common.collect.TransformedIterator.next(TransformedIterator.java:48) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Iterators.addAll(Iterators.java:357) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Lists.newArrayList(Lists.java:147) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Lists.newArrayList(Lists.java:129) ~[guava-16.0.1.jar:?]

at io.druid.query.GroupByParallelQueryRunner.run(GroupByParallelQueryRunner.java:92) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.MetricsEmittingQueryRunner$1.accumulate(MetricsEmittingQueryRunner.java:102) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.spec.SpecificSegmentQueryRunner$2$1.call(SpecificSegmentQueryRunner.java:85) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.spec.SpecificSegmentQueryRunner.doNamed(SpecificSegmentQueryRunner.java:169) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.spec.SpecificSegmentQueryRunner.access$400(SpecificSegmentQueryRunner.java:39) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.spec.SpecificSegmentQueryRunner$2.doItNamed(SpecificSegmentQueryRunner.java:160) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.spec.SpecificSegmentQueryRunner$2.accumulate(SpecificSegmentQueryRunner.java:78) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.GroupByParallelQueryRunner$1$1.call(GroupByParallelQueryRunner.java:116) ~[druid-processing-0.7.0.jar:0.7.0]

at io.druid.query.GroupByParallelQueryRunner$1$1.call(GroupByParallelQueryRunner.java:106) ~[druid-processing-0.7.0.jar:0.7.0]

at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_45]

at io.druid.query.PrioritizedExecutorService$PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:202) ~[druid-processing-0.7.0.jar:0.7.0]

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_45]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_45]

at java.lang.Thread.run(Thread.java:745) [?:1.8.0_45]

2015-07-31T02:22:38,478 INFO [qtp2066919721-41] com.metamx.emitter.core.LoggingEmitter - Event [{"feed":"alerts","timestamp":"2015-07-31T02:22:38.478-05:00","service":"realtime","host":"10.13.4.45:8099","severity":"component-failure","description":"Exception handling request","data":{"class":"io.druid.server.QueryResource","exceptionType":"java.lang.IndexOutOfBoundsException","exceptionMessage":null,"exceptionStackTrace":"java.lang.IndexOutOfBoundsException\n\tat java.nio.Buffer.checkIndex(Buffer.java:546)\n\tat java.nio.DirectByteBuffer.getInt(DirectByteBuffer.java:681)\n\tat io.druid.segment.data.VSizeIndexedInts.get(VSizeIndexedInts.java:119)\n\tat io.druid.segment.column.SimpleDictionaryEncodedColumn.getSingleValueRow(SimpleDictionaryEncodedColumn.java:61)\n\tat io.druid.segment.QueryableIndexStorageAdapter$CursorSequenceBuilder$1$1$2$1.iterator(QueryableIndexStorageAdapter.java:345)\n\tat io.druid.query.groupby.GroupByQueryEngine$RowUpdater.updateValues(GroupByQueryEngine.java:189)\n\tat io.druid.query.groupby.GroupByQueryEngine$RowUpdater.access$100(GroupByQueryEngine.java:142)\n\tat io.druid.query.groupby.GroupByQueryEngine$RowIterator.next(GroupByQueryEngine.java:367)\n\tat io.druid.query.groupby.GroupByQueryEngine$RowIterator.next(GroupByQueryEngine.java:285)\n\tat com.metamx.common.guava.BaseSequence.makeYielder(BaseSequence.java:104)\n\tat com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:81)\n\tat com.metamx.common.guava.BaseSequence.accumulate(BaseSequence.java:67)\n\tat com.metamx.common.guava.ConcatSequence$1.accumulate(ConcatSequence.java:46)\n\tat com.metamx.common.guava.ConcatSequence$1.accumulate(ConcatSequence.java:42)\n\tat com.metamx.common.guava.YieldingAccumulators$1.accumulate(YieldingAccumulators.java:32)\n\tat com.metamx.common.guava.MappingYieldingAccumulator.accumulate(MappingYieldingAccumulator.java:57)\n\tat com.metamx.common.guava.FilteringYieldingAccumulator.accumulate(FilteringYieldingAccumulator.java:69)\n\tat com.metamx.common.guava.MappingYieldingAccumulator.accumulate(MappingYieldingAccumulator.java:57)\n\tat com.metamx.common.guava.BaseSequence.makeYielder(BaseSequence.java:104)\n\tat com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:81)\n\tat com.metamx.common.guava.MappedSequence.toYielder(MappedSequence.java:46)\n\tat com.metamx.common.guava.ResourceClosingSequence.toYielder(ResourceClosingSequence.java:41)\n\tat com.metamx.common.guava.FilteredSequence.toYielder(FilteredSequence.java:52)\n\tat com.metamx.common.guava.MappedSequence.toYielder(MappedSequence.java:46)\n\tat com.metamx.common.guava.ResourceClosingSequence.toYielder(ResourceClosingSequence.java:41)\n\tat com.metamx.common.guava.YieldingSequenceBase.accumulate(YieldingSequenceBase.java:34)\n\tat com.metamx.common.guava.ConcatSequence.accumulate(ConcatSequence.java:40)\n\tat io.druid.query.GroupByParallelQueryRunner$1$1.call(GroupByParallelQueryRunner.java:116)\n\tat io.druid.query.GroupByParallelQueryRunner$1$1.call(GroupByParallelQueryRunner.java:106)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)\n\tat java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134)\n\tat com.google.common.util.concurrent.AbstractListeningExecutorService.submit(AbstractListeningExecutorService.java:58)\n\tat io.druid.query.GroupByParallelQueryRunner$1.apply(GroupByParallelQueryRunner.java:104)\n\tat io.druid.query.GroupByParallelQueryRunner$1.apply(GroupByParallelQueryRunner.java:96)\n\tat com.google.common.collect.Iterators$8.transform(Iterators.java:794)\n\tat com.google.common.collect.TransformedIterator.next(TransformedIterator.java:48)\n\tat com.google.common.collect.Iterators.addAll(Iterators.java:357)\n\tat com.google.common.collect.Lists.newArrayList(Lists.java:147)\n\tat com.google.common.collect.Lists.newArrayList(Lists.java:129)\n\tat io.druid.query.GroupByParallelQueryRunner.run(GroupByParallelQueryRunner.java:92)\n\tat io.druid.query.MetricsEmittingQueryRunner$1.accumulate(MetricsEmittingQueryRunner.java:102)\n\tat io.druid.query.spec.SpecificSegmentQueryRunner$2$1.call(SpecificSegmentQueryRunner.java:85)\n\tat io.druid.query.spec.SpecificSegmentQueryRunner.doNamed(SpecificSegmentQueryRunner.java:169)\n\tat io.druid.query.spec.SpecificSegmentQueryRunner.access$400(SpecificSegmentQueryRunner.java:39)\n\tat io.druid.query.spec.SpecificSegmentQueryRunner$2.doItNamed(SpecificSegmentQueryRunner.java:160)\n\tat io.druid.query.spec.SpecificSegmentQueryRunner$2.accumulate(SpecificSegmentQueryRunner.java:78)\n\tat io.druid.query.GroupByParallelQueryRunner$1$1.call(GroupByParallelQueryRunner.java:116)\n\tat io.druid.query.GroupByParallelQueryRunner$1$1.call(GroupByParallelQueryRunner.java:106)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat io.druid.query.PrioritizedExecutorService$PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:202)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n","exception":"java.lang.IndexOutOfBoundsException","query":"GroupByQuery{limitSpec=NoopLimitSpec, dimFilter=(product = 动态应用平台 && (httpcode = 200 || httpcode = 304 || httpcode = 403 || httpcode = 404 || RegexDimFilter{dimension='httpcode', pattern='^5.*'}) && metric = product.sip.domain.httpcode.hits.bits), granularity=AllGranularity, dimensions=[DefaultDimensionSpec{dimension='httpcode', outputName='httpcode'}], aggregatorSpecs=[LongSumAggregatorFactory{fieldName='hits', name='hits'}, LongSumAggregatorFactory{fieldName='bits', name='bits'}], postAggregatorSpecs=[], limitFn=identity}","peer":"10.209.71.150"}}]

Fangjin Yang

unread,

Aug 2, 2015, 11:28:41 AM8/2/15

to Druid User

Hi, what is the query you are issuing to cause the error? Do you mind seeing if you can reproduce this error with the latest stable of Druid?

贺祥

unread,

Aug 4, 2015, 6:27:01 AM8/4/15

to druid...@googlegroups.com

Exception in thread "plumber_persist_9" java.nio.BufferUnderflowException

at java.nio.DirectByteBuffer.get(DirectByteBuffer.java:271)

at java.nio.ByteBuffer.get(ByteBuffer.java:715)

at io.druid.common.utils.SerializerUtils.readString(SerializerUtils.java:71)

at io.druid.segment.IndexMerger.makeIndexFiles(IndexMerger.java:761)

at io.druid.segment.IndexMerger.merge(IndexMerger.java:319)

at io.druid.segment.IndexMerger.persist(IndexMerger.java:181)

at io.druid.segment.IndexMerger.persist(IndexMerger.java:151)

at io.druid.segment.IndexMerger.persist(IndexMerger.java:134)

at io.druid.segment.realtime.plumber.RealtimePlumber.persistHydrant(RealtimePlumber.java:830)

at io.druid.segment.realtime.plumber.RealtimePlumber$3.doRun(RealtimePlumber.java:354)

at io.druid.common.guava.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:40)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)

at java.lang.Thread.run(Thread.java:745)

May I ask what impact this error will cause? Will affect the query?

--
You received this message because you are subscribed to a topic in the Google Groups "Druid User" group.
To unsubscribe from this topic, visit https://groups.google.com/d/topic/druid-user/GUPSc4opHP8/unsubscribe.
To unsubscribe from this group and all its topics, send an email to druid-user+...@googlegroups.com.
To post to this group, send email to druid...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/druid-user/ca7c621c-9a6d-4d3b-9f33-cdb4e58540ac%40googlegroups.com.

For more options, visit https://groups.google.com/d/optout.

Fangjin Yang

unread,

Aug 4, 2015, 12:15:58 PM8/4/15

to Druid User

Hi 贺祥, what version of Druid is this?

To unsubscribe from this group and all its topics, send an email to druid-user+unsubscribe@googlegroups.com.

To post to this group, send email to druid...@googlegroups.com.

hexi...@gmail.com

unread,

Aug 5, 2015, 10:45:29 PM8/5/15

to Druid User

0.7.0

在 2015年8月5日星期三 UTC+8上午12:15:58，Fangjin Yang写道：

To unsubscribe from this group and all its topics, send an email to druid-user+...@googlegroups.com.

To post to this group, send email to druid...@googlegroups.com.

Gian Merlino

unread,

Aug 6, 2015, 2:14:03 PM8/6/15

to Druid User

The error messages make it look like one of the intermediate segments on disk was truncated. Can you post the logs of the process that created the problematic segment? Any "ERROR" level logs would be especially interesting.

--
You received this message because you are subscribed to the Google Groups "Druid User" group.
To unsubscribe from this group and stop receiving emails from it, send an email to druid-user+...@googlegroups.com.

To post to this group, send email to druid...@googlegroups.com.

To view this discussion on the web visit https://groups.google.com/d/msgid/druid-user/071748d1-c695-48dd-818b-6cd5e84cc6de%40googlegroups.com.

hexi...@gmail.com

unread,

Aug 7, 2015, 3:04:25 AM8/7/15

to Druid User

Thank you very much for your answer, I have two questions, the first question, my node realtime non normal exit, when I started again, it will restart the consumer Kafka, but I want to reset offset, so that node realtime from the current time to start spending, what I need to do?

The second question, if I start two node realtime to consume the same topic Kafka, just start the time will not be a problem, but the operation will happen for some time. Consumer data is not normal, the query will be abnormal.

Please see my configuration

realtime node runtime.properties

druid.host=XXXXXXXXXXXXXX

#druid.service=realtime

druid.port=8099

druid.service=realtime

#druid.service=druid/prod/realtime

druid.zk.service.host=XXXXXXXXXXXXXXXXXX

#druid.extensions.coordinates=["io.druid.extensions:druid-kafka-seven:0.6.171"]

druid.extensions.coordinates=["io.druid.extensions:druid-kafka-eight","io.druid.extensions:druid-hdfs-storage:0.7.0","io.druid.extensions:mysql-metadata-storage"]

#druid.extensions.coordinates=["io.druid.extensions:druid-kafka-eight","io.druid.extensions:mysql-metadata-storage"]

hadoopDependencyCoordinates=["org.apache.hadoop:hadoop-client:2.3.0-cdh5.0.0"]

# Change this config to db to hand off to the rest of the Druid cluster

#druid.publish.type=noop

#druid.publish.type=db

# These configs are only required for real hand off

#druid.db.connector.connectURI=XXXXXXXXXXXXXXXX

#druid.db.connector.user=druid

#druid.db.connector.password=druid

#druid.db.connector.useValidationQuery=true

#druid.db.tables.base=prod

#newer 0.7

druid.metadata.storage.connector.connectURI=XXXXXXXXXXXXX

druid.metadata.storage.connector.user=XXXXXXXXXX

druid.metadata.storage.connector.password=XXXXXXXXX

druid.metadata.storage.connector.useValidationQuery=true

druid.metadata.storage.type=mysql

druid.processing.buffer.sizeBytes=100000000

#druid.processing.buffer.sizeBytes=2000000

druid.processing.numThreads=12

#druid.storage.type=local

#druid.storage.storageDirectory=/data1/druid/realtimeStore

##

druid.storage.type=hdfs

druid.storage.storageDirectory=hdfs\://eosdip/user/druid/normal/storage

#vm23787.xjf.sce.sina.com

druid.pusher.hdfs=true

druid.pusher.hdfs.storageDirectory=hdfs\://eosdip/user/druid/normal/pusher

druid.monitoring.monitors=["com.metamx.metrics.JvmMonitor","io.druid.segment.realtime.RealtimeMetricsMonitor"]

druid.emitter=logging

*.spec

[

{

"dataSchema" : {

"dataSource" : "www-analyze-edge",

"parser" : {

"type" : "string",

"parseSpec" : {

"format" : "json",

"timestampSpec" : {

"column" : "timestamp",

"format" : "posix"

},

"dimensionsSpec" : {

"dimensions":

["product","sip","httpcode","domain","metric","province","sizerank","filetyperank","speedrank","timerank","category","idc","isp","hitormiss"],

"dimensionExclusions" :[] ,

"spatialDimensions" : []

}

},

"metricsSpec" : [{

"type" : "count",

"name" : "count"

}

, {

"type" : "longSum",

"name" : "hits",

"fieldName" : "hits"

}

, {

"type" : "longSum",

"name" : "bits",

"fieldName" : "bits"

}

],

"granularitySpec" : {

"type" : "uniform",

"segmentGranularity" : "hour",

"queryGranularity" : "minute"

}

},

"ioConfig" : {

"type" : "realtime",

"firehose": {

"type": "kafka-0.8",

"consumerProps": {

"zookeeper.connect": "XXXXXXXXXXXXXXXXXXX",

"zookeeper.connection.timeout.ms" : "15000",

"zookeeper.session.timeout.ms" : "15000",

"zookeeper.sync.time.ms" : "5000",

"group.id": "www-edge",

"fetch.message.max.bytes" : "1048586",

"auto.offset.reset": "largest",

"auto.commit.enable": "true"

},

"feed": "XXXXXXXXXXXXX"

},

"plumber": {

"type": "realtime"

}

},

"tuningConfig": {

"type" : "realtime",

"maxRowsInMemory": 5000000,

"intermediatePersistPeriod": "PT2m",

"windowPeriod": "PT50m",

"basePersistDirectory": "\/data0\/tmp\/realtime\/basePersist",

"rejectionPolicy": {

"type": "serverTime"

},

"shardSpec": {

"type": "linear",

"partitionNum": 2

}

]

在 2015年8月7日星期五 UTC+8上午2:14:03，Gian Merlino写道：

Nishant Bangarwa

unread,

Aug 8, 2015, 3:34:03 AM8/8/15

to druid...@googlegroups.com

On Fri, Aug 7, 2015 at 12:34 PM, <hexi...@gmail.com> wrote:

Thank you very much for your answer, I have two questions, the first question, my node realtime non normal exit, when I started again, it will restart the consumer Kafka, but I want to reset offset, so that node realtime from the current time to start spending, what I need to do?

you have already set "auto.offset.reset" to largest in consumerProps for kafkaFirehose. It will automatically reset offset when a node is restarted.

The second question, if I start two node realtime to consume the same topic Kafka, just start the time will not be a problem, but the operation will happen for some time. Consumer data is not normal, the query will be abnormal.

In case of sharding both realtime nodes should have different shard numbers in their shardSpec.

make sure that shard numbers are unique.

To view this discussion on the web visit https://groups.google.com/d/msgid/druid-user/d5de47ab-f12a-4d86-ab88-4f9576059c48%40googlegroups.com.

For more options, visit https://groups.google.com/d/optout.

--

Nishant

Software Engineer

|

METAMARKETS

m	+91-9729200044

nishant....@metamarkets.com

Reply all

Reply to author

Forward