realtime with kafka 8 will not start

[David Montgomery]

Hi,

I dont get how to fix my realtime spec file using the documentation online.  I am using kafka 8, the firehose in the docs and yet I get an error that I cant interpret.  How do I resolve given errors?

/usr/bin/java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.realtime.specFile=/var/realtime.spec -classpath /var/druid/lib/*:/var/druid/config/realtime io.druid.cli.Main server realtime

[{
  "schema" : { "dataSource":"bg-topic-feedtest",
               "aggregators":[ {"type":"count", "name":"impressions"}],
               "indexGranularity":"minute",
           "shardSpec" : { "type": "none" } },
  "config" : { "maxRowsInMemory" : 500000,
               "intermediatePersistPeriod" : "PT10m" },
 "firehose" : {
        "type" : "kafka-0.8",
        "consumerProps" : {
            "zookeeper.connect": "localhost:2181",
            "zookeeper.connection.timeout.ms": "15000",
            "zookeeper.session.timeout.ms": "15000",
            "zookeeper.sync.time.ms": "5000",
            "group.id": "topic-pixel-local",
            "fetch.message.max.bytes": "1048586",
            "auto.offset.reset": "largest",
            "auto.commit.enable": "false"
        },
        "feed" : "druidtest",
        "parser" : {
            "timestampSpec" : {
                "column" : "utcdt",
                "format" : "iso"
            },
            "data" : {
                "format" : "json"
            },
            "dimensionExclusions" : [
                "wp"
            ]
        }
    },
   
    "plumber" : {
        "type" : "realtime",
        "windowPeriod" : "PT10m",
        "segmentGranularity":"hour",
        "basePersistDirectory" : "/tmp/realtime/basePersist",
        "rejectionPolicyFactory": {
            "type": "messageTime"
        }
    }
]   


druid.host=localhost
druid.service=realtime
druid.port=8083

druid.zk.service.host=<%=@zookeeper%>

druid.extensions.coordinates=["io.druid.extensions:druid-kafka-eight:0.6.143"]

# Change this config to db to hand off to the rest of the Druid cluster
druid.publish.type=noop

# These configs are only required for real hand off
druid.db.connector.connectURI=jdbc\:mysql\://<%=@mysql_host%>\:3306/<%=@mysql_database%>
druid.db.connector.user=<%=@mysql_username%>
druid.db.connector.password=<%=@mysql_password%>

druid.processing.buffer.sizeBytes=100000000
druid.processing.numThreads=1

druid.monitoring.monitors=["io.druid.segment.realtime.RealtimeMetricsMonitor"]

druid.storage.type=s3
druid.storage.bucket=<%=@s3bucket%>
druid.storage.baseKey=<%=@s3basekey%>

druid.realtime.specFile=/var/realtime.spec






Exception in thread "main" com.google.inject.CreationException: Guice creation errors:

1) Error injecting constructor, java.lang.NoClassDefFoundError: com/metamx/common/parsers/ParseException
  at io.druid.guice.FireDepartmentsProvider.<init>(FireDepartmentsProvider.java:43)
  while locating io.druid.guice.FireDepartmentsProvider
  at io.druid.guice.RealtimeModule.configure(RealtimeModule.java:79)
  while locating java.util.List<io.druid.segment.realtime.FireDepartment>
    for parameter 0 at io.druid.segment.realtime.RealtimeMetricsMonitor.<init>(RealtimeMetricsMonitor.java:40)
  while locating io.druid.segment.realtime.RealtimeMetricsMonitor
  at io.druid.server.metrics.MetricsModule.getMonitorScheduler(MetricsModule.java:80)
  at io.druid.server.metrics.MetricsModule.getMonitorScheduler(MetricsModule.java:80)
  while locating com.metamx.metrics.MonitorScheduler
  at io.druid.server.metrics.MetricsModule.configure(MetricsModule.java:65)
  while locating com.metamx.metrics.MonitorScheduler annotated with @com.google.inject.name.Named(value=ForTheEagerness)

1 error
    at com.google.inject.internal.Errors.throwCreationExceptionIfErrorsExist(Errors.java:448)
    at com.google.inject.internal.InternalInjectorCreator.injectDynamically(InternalInjectorCreator.java:184)
    at com.google.inject.internal.InternalInjectorCreator.build(InternalInjectorCreator.java:110)
    at com.google.inject.Guice.createInjector(Guice.java:96)
    at com.google.inject.Guice.createInjector(Guice.java:73)
    at com.google.inject.Guice.createInjector(Guice.java:62)
    at io.druid.initialization.Initialization.makeInjectorWithModules(Initialization.java:346)
    at io.druid.cli.GuiceRunnable.makeInjector(GuiceRunnable.java:56)
    at io.druid.cli.ServerRunnable.run(ServerRunnable.java:39)
    at io.druid.cli.Main.main(Main.java:92)
Caused by: java.lang.NoClassDefFoundError: com/metamx/common/parsers/ParseException
    at java.lang.Class.getDeclaredMethods0(Native Method)
    at java.lang.Class.privateGetDeclaredMethods(Class.java:2451)
    at java.lang.Class.getDeclaredMethods(Class.java:1810)
    at com.fasterxml.jackson.databind.introspect.AnnotatedClass._addMemberMethods(AnnotatedClass.java:593)
    at com.fasterxml.jackson.databind.introspect.AnnotatedClass.resolveMemberMethods(AnnotatedClass.java:416)
    at com.fasterxml.jackson.databind.introspect.AnnotatedClass.memberMethods(AnnotatedClass.java:243)
    at com.fasterxml.jackson.databind.introspect.POJOPropertiesCollector._addMethods(POJOPropertiesCollector.java:457)
    at com.fasterxml.jackson.databind.introspect.POJOPropertiesCollector.collect(POJOPropertiesCollector.java:233)
    at com.fasterxml.jackson.databind.introspect.BasicClassIntrospector.collectProperties(BasicClassIntrospector.java:142)
    at com.fasterxml.jackson.databind.introspect.BasicClassIntrospector.forDeserialization(BasicClassIntrospector.java:81)
    at com.fasterxml.jackson.databind.introspect.BasicClassIntrospector.forDeserialization(BasicClassIntrospector.java:11)
    at com.fasterxml.jackson.databind.DeserializationConfig.introspect(DeserializationConfig.java:507)
    at com.fasterxml.jackson.databind.deser.DeserializerCache._createDeserializer(DeserializerCache.java:329)
    at com.fasterxml.jackson.databind.deser.DeserializerCache._createAndCache2(DeserializerCache.java:267)
    at com.fasterxml.jackson.databind.deser.DeserializerCache._createAndCacheValueDeserializer(DeserializerCache.java:247)
    at com.fasterxml.jackson.databind.deser.DeserializerCache.findValueDeserializer(DeserializerCache.java:146)
    at com.fasterxml.jackson.databind.DeserializationContext.findContextualValueDeserializer(DeserializationContext.java:305)
    at com.fasterxml.jackson.databind.jsontype.impl.TypeDeserializerBase._findDeserializer(TypeDeserializerBase.java:173)
    at com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId(AsPropertyTypeDeserializer.java:99)
    at com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject(AsPropertyTypeDeserializer.java:82)
    at com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType(AbstractDeserializer.java:106)
    at com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize(SettableBeanProperty.java:462)
    at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:347)
    at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:977)
    at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:276)
    at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:121)
    at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:230)
    at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:207)
    at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:23)
    at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:2888)
    at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:1995)
    at io.druid.guice.FireDepartmentsProvider.<init>(FireDepartmentsProvider.java:45)
    at io.druid.guice.FireDepartmentsProvider$$FastClassByGuice$$229da177.newInstance(<generated>)
    at com.google.inject.internal.cglib.reflect.$FastConstructor.newInstance(FastConstructor.java:40)
    at com.google.inject.internal.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:61)
    at com.google.inject.internal.ConstructorInjector.provision(ConstructorInjector.java:108)
    at com.google.inject.internal.ConstructorInjector.construct(ConstructorInjector.java:88)
    at com.google.inject.internal.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:269)
    at com.google.inject.internal.BoundProviderFactory.get(BoundProviderFactory.java:62)
    at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46)
    at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1058)
    at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40)
    at com.google.inject.Scopes$1$1.get(Scopes.java:65)
    at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:41)
    at com.google.inject.internal.SingleParameterInjector.inject(SingleParameterInjector.java:38)
    at com.google.inject.internal.SingleParameterInjector.getAll(SingleParameterInjector.java:62)
    at com.google.inject.internal.ConstructorInjector.provision(ConstructorInjector.java:107)
    at com.google.inject.internal.ConstructorInjector.construct(ConstructorInjector.java:88)
    at com.google.inject.internal.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:269)
    at com.google.inject.internal.InjectorImpl$3$1.call(InjectorImpl.java:1005)
    at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1058)
    at com.google.inject.internal.InjectorImpl$3.get(InjectorImpl.java:1001)
    at com.google.inject.internal.InjectorImpl.getInstance(InjectorImpl.java:1040)
    at io.druid.server.metrics.MetricsModule.getMonitorScheduler(MetricsModule.java:83)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:601)
    at com.google.inject.internal.ProviderMethod.get(ProviderMethod.java:105)
    at com.google.inject.internal.ProviderInternalFactory.provision(ProviderInternalFactory.java:86)
    at com.google.inject.internal.InternalFactoryToInitializableAdapter.provision(InternalFactoryToInitializableAdapter.java:55)
    at com.google.inject.internal.ProviderInternalFactory.circularGet(ProviderInternalFactory.java:66)
    at com.google.inject.internal.InternalFactoryToInitializableAdapter.get(InternalFactoryToInitializableAdapter.java:47)
    at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46)
    at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1058)
    at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40)
    at io.druid.guice.LifecycleScope$1.get(LifecycleScope.java:49)
    at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:41)
    at com.google.inject.internal.FactoryProxy.get(FactoryProxy.java:56)
    at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46)
    at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1058)
    at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40)
    at com.google.inject.Scopes$1$1.get(Scopes.java:65)
    at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:41)
    at com.google.inject.internal.InternalInjectorCreator$1.call(InternalInjectorCreator.java:205)
    at com.google.inject.internal.InternalInjectorCreator$1.call(InternalInjectorCreator.java:199)
    at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1051)
    at com.google.inject.internal.InternalInjectorCreator.loadEagerSingletons(InternalInjectorCreator.java:199)
    at com.google.inject.internal.InternalInjectorCreator.injectDynamically(InternalInjectorCreator.java:180)
    ... 8 more
Caused by: java.lang.ClassNotFoundException: com.metamx.common.parsers.ParseException
    at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:423)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:356)
    ... 87 more

[Deepak Jain]

Please refer : https://groups.google.com/forum/#!searchin/druid-development/informative$20post$20$20Deepak/druid-development/hm1xRwCavn0/FOCzUjhp3r4J

Can you make sure the missing classes jar is in cp.