Need to contract for Red5 expert to identify an fix problem

[Steve Ricketts]

I have been running a Red5 application for over 5 years on multiple servers.  This past week, the application just quit working.  Red5 looked like it was still running but it wouldn't respond to any live or recorded requests.  It did this on two consecutive days. 

The application, eTutor, serves both live and recordings of live training sessions.  It is written with Flex 4.6.

We were running 1.0RC1 on this particular server.  We upgraded to 1.0.5 and it failed again last night in the same way.  The only change to the system has been the application of windows updates.  We will remove those updates but it seems unlikely that those might be the problem.

We have no clue as to what's happening.  We do have the following from the error logs on the 1.0RC1 verison when it failed.  The error log for the 1.0.5 version was overwritten when we restarted Red5 so I don't have anything there.

2015-06-25 13:46:34,032 [pool-2334-thread-29] ERROR o.r.s.n.r.codec.RTMPProtocolEncoder - Error encoding object: 

java.util.ConcurrentModificationException: null

at java.util.HashMap$HashIterator.nextEntry(HashMap.java:839) ~[na:1.7.0_67]

at java.util.HashMap$EntryIterator.next(HashMap.java:880) ~[na:1.7.0_67]

at java.util.HashMap$EntryIterator.next(HashMap.java:878) ~[na:1.7.0_67]

at org.red5.io.amf.Output.writeObject(Output.java:450) ~[red5.jar:na]

at org.red5.io.object.Serializer.writeObjectType(Serializer.java:290) ~[red5.jar:na]

at org.red5.io.object.Serializer.writeComplex(Serializer.java:140) ~[red5.jar:na]

at org.red5.io.object.Serializer.serialize(Serializer.java:86) ~[red5.jar:na]

at org.red5.io.object.Serializer.serialize(Serializer.java:63) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolEncoder.doEncodeSharedObject(RTMPProtocolEncoder.java:684) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolEncoder.encodeSharedObject(RTMPProtocolEncoder.java:614) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolEncoder.encodeMessage(RTMPProtocolEncoder.java:553) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolEncoder.encodePacket(RTMPProtocolEncoder.java:155) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolEncoder.encode(RTMPProtocolEncoder.java:123) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPMinaProtocolEncoder.encode(RTMPMinaProtocolEncoder.java:59) [red5.jar:na]

at org.apache.mina.filter.codec.ProtocolCodecFilter.filterWrite(ProtocolCodecFilter.java:322) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.callPreviousFilterWrite(DefaultIoFilterChain.java:509) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.access$1400(DefaultIoFilterChain.java:46) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain$EntryImpl$1.filterWrite(DefaultIoFilterChain.java:808) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain$TailFilter.filterWrite(DefaultIoFilterChain.java:734) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.callPreviousFilterWrite(DefaultIoFilterChain.java:509) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.fireFilterWrite(DefaultIoFilterChain.java:501) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.session.AbstractIoSession.write(AbstractIoSession.java:490) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.session.AbstractIoSession.write(AbstractIoSession.java:435) [mina-core-2.0.2.jar:na]

at org.red5.server.net.rtmp.RTMPMinaConnection.write(RTMPMinaConnection.java:273) [red5.jar:na]

at org.red5.server.net.rtmp.Channel.write(Channel.java:124) [red5.jar:na]

at org.red5.server.net.rtmp.Channel.write(Channel.java:104) [red5.jar:na]

at org.red5.server.so.SharedObject$1.run(SharedObject.java:354) [red5.jar:na]

at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [na:1.7.0_67]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [na:1.7.0_67]

at java.lang.Thread.run(Unknown Source) [na:1.7.0_67]

2015-06-25 13:47:53,701 [NioProcessor-1] ERROR o.r.s.n.r.codec.RTMPProtocolDecoder - Error decoding buffer

org.red5.server.net.protocol.ProtocolException: Error during decoding

at org.red5.server.net.rtmp.codec.RTMPProtocolDecoder.decode(RTMPProtocolDecoder.java:197) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolDecoder.decodeBuffer(RTMPProtocolDecoder.java:118) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPMinaProtocolDecoder.decode(RTMPMinaProtocolDecoder.java:62) [red5.jar:na]

at org.apache.mina.filter.codec.ProtocolCodecFilter.messageReceived(ProtocolCodecFilter.java:241) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.callNextMessageReceived(DefaultIoFilterChain.java:434) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.access$1200(DefaultIoFilterChain.java:46) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain$EntryImpl$1.messageReceived(DefaultIoFilterChain.java:796) [mina-core-2.0.2.jar:na]

at org.red5.server.net.rtmpe.RTMPEIoFilter.messageReceived(RTMPEIoFilter.java:125) [red5.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.callNextMessageReceived(DefaultIoFilterChain.java:434) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.access$1200(DefaultIoFilterChain.java:46) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain$EntryImpl$1.messageReceived(DefaultIoFilterChain.java:796) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.IoFilterAdapter.messageReceived(IoFilterAdapter.java:119) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.callNextMessageReceived(DefaultIoFilterChain.java:434) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.fireMessageReceived(DefaultIoFilterChain.java:426) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.polling.AbstractPollingIoProcessor.read(AbstractPollingIoProcessor.java:693) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.polling.AbstractPollingIoProcessor.process(AbstractPollingIoProcessor.java:646) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.polling.AbstractPollingIoProcessor.process(AbstractPollingIoProcessor.java:635) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.polling.AbstractPollingIoProcessor.access$400(AbstractPollingIoProcessor.java:67) [mina-core-2.0.2.jar:na]

at org.apache.mina.core.polling.AbstractPollingIoProcessor$Processor.run(AbstractPollingIoProcessor.java:1079) [mina-core-2.0.2.jar:na]

at org.apache.mina.util.NamePreservingRunnable.run(NamePreservingRunnable.java:64) [mina-core-2.0.2.jar:na]

at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [na:1.7.0_67]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [na:1.7.0_67]

at java.lang.Thread.run(Unknown Source) [na:1.7.0_67]

Caused by: java.nio.BufferUnderflowException: null

at java.nio.Buffer.nextGetIndex(Unknown Source) ~[na:1.7.0_67]

at java.nio.HeapByteBuffer.get(Unknown Source) ~[na:1.7.0_67]

at org.apache.mina.core.buffer.AbstractIoBuffer.get(AbstractIoBuffer.java:492) ~[mina-core-2.0.2.jar:na]

at org.red5.io.amf3.Input.readAMF3Integer(Input.java:944) ~[red5.jar:na]

at org.red5.io.amf3.Input.readNumber(Input.java:340) ~[red5.jar:na]

at org.red5.io.object.Deserializer.deserialize(Deserializer.java:70) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolDecoder.decodeNotifyOrInvoke(RTMPProtocolDecoder.java:859) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolDecoder.decodeInvoke(RTMPProtocolDecoder.java:753) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolDecoder.decodeMessage(RTMPProtocolDecoder.java:525) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolDecoder.decodePacket(RTMPProtocolDecoder.java:358) ~[red5.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolDecoder.decode(RTMPProtocolDecoder.java:184) ~[red5.jar:na]

... 22 common frames omitted

We have very upset customers since Red5 is used for both live and VOD on a daily basis and used by about 1,800 people a day.  We urgently need an expert to come in and look at the application and see if they can find out what's happening.  Please contact me at sricketts at learn.net or post here if you are interested in contracting for this project.

[Mondain]

Your error with the rc version is fixed in the 1.0.x versions. I'll email you when I get home on Tuesday, unless you've already resolved by then.

--

---
You received this message because you are subscribed to the Google Groups "red5" group.
To unsubscribe from this group and stop receiving emails from it, send an email to red5interest...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[Steve Ricketts]

I "think" it happened again with 1.0.5 with 64 bit JDK 1.8.  I say think because we restarted Red5 quickly and lost the logs.  The other puzzler is why did it happen all of a sudden, two days in a row without anything changing.   Look forward to your e-mail on Tuesday, thanks!

[Steve Ricketts]

Ok, have 1.0.5 running with 64 bit JDK 1.8 from red5.bat on Server 2008 R2.  I recompiled the web app class using the 1.0.5 Red5-server-common.jar and Red5-server.jar.  Getting the same "Error encoding object" errors but haven't hit the "error decoding buffer" error that stops Red5.  Here's an example of the first error:

2015-06-29 14:03:33,971 [SharedObjectScheduler-1] ERROR o.r.s.n.r.codec.RTMPProtocolEncoder - Error encoding

java.util.ConcurrentModificationException: null

at java.util.HashMap$HashIterator.nextNode(HashMap.java:1429) ~[na:1.8.0_45]

at java.util.HashMap$EntryIterator.next(HashMap.java:1463) ~[na:1.8.0_45]

at java.util.HashMap$EntryIterator.next(HashMap.java:1461) ~[na:1.8.0_45]

at org.red5.io.amf3.Output.writeObject(Output.java:490) ~[red5-io-1.0.5-RELEASE.jar:na]

at org.red5.io.object.Serializer.writeObjectType(Serializer.java:324) ~[red5-io-1.0.5-RELEASE.jar:na]

at org.red5.io.object.Serializer.writeComplex(Serializer.java:175) ~[red5-io-1.0.5-RELEASE.jar:na]

at org.red5.io.object.Serializer.serialize(Serializer.java:123) ~[red5-io-1.0.5-RELEASE.jar:na]

at org.red5.io.object.Serializer.serialize(Serializer.java:64) ~[red5-io-1.0.5-RELEASE.jar:na]

at org.red5.server.net.rtmp.codec.RTMPProtocolEncoder.doEncodeSharedObject(RTMPProtocolEncoder.java:665) ~[red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.red5.server.net.rtmp.codec.RTMPProtocolEncoder.encodeFlexSharedObject(RTMPProtocolEncoder.java:590) ~[red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.red5.server.net.rtmp.codec.RTMPProtocolEncoder.encodeMessage(RTMPProtocolEncoder.java:536) ~[red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.red5.server.net.rtmp.codec.RTMPProtocolEncoder.encodePacket(RTMPProtocolEncoder.java:134) ~[red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.red5.server.net.rtmp.codec.RTMPProtocolEncoder.encode(RTMPProtocolEncoder.java:109) ~[red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.red5.server.net.rtmp.codec.RTMPMinaProtocolEncoder.encode(RTMPMinaProtocolEncoder.java:69) [red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.apache.mina.filter.codec.ProtocolCodecFilter.filterWrite(ProtocolCodecFilter.java:306) [mina-core-2.0.8.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.callPreviousFilterWrite(DefaultIoFilterChain.java:625) [mina-core-2.0.8.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.access$1500(DefaultIoFilterChain.java:48) [mina-core-2.0.8.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain$EntryImpl$1.filterWrite(DefaultIoFilterChain.java:953) [mina-core-2.0.8.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain$TailFilter.filterWrite(DefaultIoFilterChain.java:877) [mina-core-2.0.8.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.callPreviousFilterWrite(DefaultIoFilterChain.java:625) [mina-core-2.0.8.jar:na]

at org.apache.mina.core.filterchain.DefaultIoFilterChain.fireFilterWrite(DefaultIoFilterChain.java:618) [mina-core-2.0.8.jar:na]

at org.apache.mina.core.session.AbstractIoSession.write(AbstractIoSession.java:502) [mina-core-2.0.8.jar:na]

at org.apache.mina.core.session.AbstractIoSession.write(AbstractIoSession.java:447) [mina-core-2.0.8.jar:na]

at org.red5.server.net.rtmp.RTMPMinaConnection.write(RTMPMinaConnection.java:328) [red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.red5.server.net.rtmp.Channel.write(Channel.java:136) [red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.red5.server.net.rtmp.Channel.write(Channel.java:106) [red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.red5.server.net.rtmp.RTMPConnection.sendSharedObjectMessage(RTMPConnection.java:1399) [red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.red5.server.so.SharedObject$2.run(SharedObject.java:366) [red5-server-common-1.0.5-RELEASE.jar:1.0.5-RELEASE]

at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) [spring-context-4.0.8.RELEASE.jar:4.0.8.RELEASE]

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_45]

at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_45]

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_45]

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_45]

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_45]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_45]

at java.lang.Thread.run(Thread.java:745) [na:1.8.0_45]